01JN461210QG0EEKMPZMBWY7F6: test-up-2region-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01JN461FMF5KAWGE96BR34NXSG

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12025-02-27T17:21:43.204Zjob dependencies complete; ready to run (waiting for 14 m 8 s)
22025-02-27T17:22:39.453Zjob assigned to worker 01JN46VKXFNF2X2BGRECDC9DHA [factory aws, i-0886b53d35e112012] (queued for 56 s)
32025-02-27T17:22:47.629Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42025-02-27T17:22:48.853Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52025-02-27T17:22:48.853Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62025-02-27T17:22:49.985Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72025-02-27T17:22:49.985Zdownloading input: /input/build/work/bins/crutest.gz
82025-02-27T17:22:51.155Zdownloaded input: /input/build/work/bins/crutest.gz
92025-02-27T17:22:51.155Zdownloading input: /input/build/work/bins/dsc.gz
102025-02-27T17:22:51.818Zdownloaded input: /input/build/work/bins/dsc.gz
112025-02-27T17:22:51.821Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122025-02-27T17:22:51.835Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132025-02-27T17:22:51.835Zdownloading input: /input/build/work/scripts/test_ds.sh
142025-02-27T17:22:51.854Zdownloaded input: /input/build/work/scripts/test_ds.sh
152025-02-27T17:22:51.854Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162025-02-27T17:22:51.872Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172025-02-27T17:22:51.872Zdownloading input: /input/build/work/scripts/test_repair.sh
182025-02-27T17:22:51.888Zdownloaded input: /input/build/work/scripts/test_repair.sh
192025-02-27T17:22:51.888Zdownloading input: /input/build/work/scripts/test_replay.sh
202025-02-27T17:22:51.905Zdownloaded input: /input/build/work/scripts/test_replay.sh
212025-02-27T17:22:51.906Zdownloading input: /input/build/work/scripts/test_up.sh
222025-02-27T17:22:51.926Zdownloaded input: /input/build/work/scripts/test_up.sh
232025-02-27T17:22:51.926Zdownloading input: /input/build/work/scripts/upstairs_info.d
242025-02-27T17:22:51.952Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252025-02-27T17:22:51.956Zdownloading input: /input/build/tmp/cargo-test-out.log
262025-02-27T17:22:52.620Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272025-02-27T17:22:52.620Zstarting task 0: "setup"
282025-02-27T17:22:52.624Z++ uname -s
292025-02-27T17:22:52.624Z+ kern=SunOS
302025-02-27T17:22:52.624Z+ build_user=build
312025-02-27T17:22:52.624Z+ build_uid=12345
322025-02-27T17:22:52.624Z+ work_dir=/work
332025-02-27T17:22:52.624Z+ input_dir=/input
342025-02-27T17:22:52.624Z+ [[ 0 == 12345 ]]
352025-02-27T17:22:52.624Z+ case "$kern" in
362025-02-27T17:22:52.624Z+ groupadd -g 12345 build
372025-02-27T17:22:52.627Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382025-02-27T17:22:54.633Z+ zfs create -o mountpoint=/work rpool/work
392025-02-27T17:22:54.830Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402025-02-27T17:22:54.833Z+ home_fs=zfs
412025-02-27T17:22:54.833Z+ [[ zfs == autofs ]]
422025-02-27T17:22:54.833Z+ mkdir -p /home/build
432025-02-27T17:22:54.836Z+ chown build:build /home/build /work
442025-02-27T17:22:56.851Z+ chmod 0700 /home/build /work
452025-02-27T17:22:56.860Zprocess exited: duration 4227 ms, exit code 0
 
462025-02-27T17:22:56.879Zstarting task 1: "authentication"
472025-02-27T17:22:56.894Zprocess exited: duration 17 ms, exit code 0
 
482025-02-27T17:22:56.902Zstarting task 2: "build"
492025-02-27T17:22:56.905Z+ banner cores
502025-02-27T17:22:56.908Z
512025-02-27T17:22:56.908Z #### #### ##### ###### ####
522025-02-27T17:22:56.908Z # # # # # # # #
532025-02-27T17:22:56.908Z # # # # # ##### ####
542025-02-27T17:22:56.908Z # # # ##### # #
552025-02-27T17:22:56.908Z # # # # # # # # #
562025-02-27T17:22:56.908Z #### #### # # ###### ####
572025-02-27T17:22:56.909Z
582025-02-27T17:22:56.909Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
592025-02-27T17:22:56.917Z+ banner unpack
602025-02-27T17:22:56.922Z
612025-02-27T17:22:56.922Z # # # # ##### ## #### # #
622025-02-27T17:22:56.922Z # # ## # # # # # # # # #
632025-02-27T17:22:56.922Z # # # # # # # # # # ####
642025-02-27T17:22:56.922Z # # # # # ##### ###### # # #
652025-02-27T17:22:56.923Z # # # ## # # # # # # #
662025-02-27T17:22:56.923Z #### # # # # # #### # #
672025-02-27T17:22:56.923Z
682025-02-27T17:22:56.923Z+ mkdir -p /var/tmp/bins
692025-02-27T17:22:56.926Z+ for t in "$input/bins/"*.gz
702025-02-27T17:22:56.926Z++ basename /input/build/work/bins/crucible-downstairs.gz
712025-02-27T17:22:56.926Z+ b=crucible-downstairs.gz
722025-02-27T17:22:56.926Z+ b=crucible-downstairs
732025-02-27T17:22:56.926Z+ gunzip
742025-02-27T17:22:59.314Z+ chmod +x /var/tmp/bins/crucible-downstairs
752025-02-27T17:22:59.317Z+ for t in "$input/bins/"*.gz
762025-02-27T17:22:59.320Z++ basename /input/build/work/bins/crucible-hammer.gz
772025-02-27T17:22:59.320Z+ b=crucible-hammer.gz
782025-02-27T17:22:59.320Z+ b=crucible-hammer
792025-02-27T17:22:59.320Z+ gunzip
802025-02-27T17:23:01.457Z+ chmod +x /var/tmp/bins/crucible-hammer
812025-02-27T17:23:01.460Z+ for t in "$input/bins/"*.gz
822025-02-27T17:23:01.462Z++ basename /input/build/work/bins/crutest.gz
832025-02-27T17:23:01.462Z+ b=crutest.gz
842025-02-27T17:23:01.462Z+ b=crutest
852025-02-27T17:23:01.462Z+ gunzip
862025-02-27T17:23:03.915Z+ chmod +x /var/tmp/bins/crutest
872025-02-27T17:23:03.917Z+ for t in "$input/bins/"*.gz
882025-02-27T17:23:03.920Z++ basename /input/build/work/bins/dsc.gz
892025-02-27T17:23:03.920Z+ b=dsc.gz
902025-02-27T17:23:03.920Z+ b=dsc
912025-02-27T17:23:03.920Z+ gunzip
922025-02-27T17:23:05.251Z+ chmod +x /var/tmp/bins/dsc
932025-02-27T17:23:05.254Z+ export BINDIR=/var/tmp/bins
942025-02-27T17:23:05.254Z+ BINDIR=/var/tmp/bins
952025-02-27T17:23:05.254Z+ jobpid=1147
962025-02-27T17:23:05.254Z+ echo 'Setup debug logging'
972025-02-27T17:23:05.254ZSetup debug logging
982025-02-27T17:23:05.254Z+ mkdir /tmp/debug
992025-02-27T17:23:05.258Z+ sleep 3600
1002025-02-27T17:23:05.258Z+ psrinfo -v
1012025-02-27T17:23:05.261Z+ df -h
1022025-02-27T17:23:05.267Z+ prstat -d d -mLc 1
1032025-02-27T17:23:05.270Z+ iostat -T d -xn 1
1042025-02-27T17:23:05.270Z+ mpstat -T d 1
1052025-02-27T17:23:05.270Z+ vmstat -T d -p 1
1062025-02-27T17:23:05.270Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1072025-02-27T17:23:05.280Z+ banner test_up_2r_encrypted
1082025-02-27T17:23:05.280Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1092025-02-27T17:23:05.280Z #####
1102025-02-27T17:23:05.280Z ##### ###### #### ##### # # ##### # # #####
1112025-02-27T17:23:05.280Z # # # # # # # # # # #
1122025-02-27T17:23:05.280Z # ##### #### # # # # # ##### # #
1132025-02-27T17:23:05.280Z # # # # # # ##### # #####
1142025-02-27T17:23:05.280Z # # # # # # # # # # #
1152025-02-27T17:23:05.280Z # ###### #### # ####### #### # ####### ####### # #
1162025-02-27T17:23:05.281Z
1172025-02-27T17:23:05.281Z+ ptime -m bash /input/build/work/scripts/test_up.sh -r 2 -N encrypted
1182025-02-27T17:23:05.285Z/input/build/work
1192025-02-27T17:23:05.299ZUsing 2 region sets
1202025-02-27T17:23:05.299ZTurn off color for downstairs dump
1212025-02-27T17:23:05.316ZUpstairs using key: 0Md83Np50W5ygafJAJVBph2x9t42z9tuOdLMUgV6x80=
1222025-02-27T17:23:05.319Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1232025-02-27T17:23:05.319ZCreating 6 downstairs regions
1242025-02-27T17:23:05.677ZStarting 6 downstairs
1252025-02-27T17:23:05.680Zdsc started at PID: 1195
1262025-02-27T17:23:11.017ZDisable automatic restart on all downstairs
1272025-02-27T17:23:11.062Z
1282025-02-27T17:23:11.062ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1292025-02-27T17:23:11.068ZRunning test: span
1302025-02-27T17:23:15.197ZCompleted test: span
1312025-02-27T17:23:20.204ZRunning test: big
1322025-02-27T17:23:24.782ZCompleted test: big
1332025-02-27T17:23:29.789ZRunning test: dep
1342025-02-27T17:24:03.115ZCompleted test: dep
1352025-02-27T17:24:08.122ZRunning test: balloon
1362025-02-27T17:28:21.252ZCompleted test: balloon
1372025-02-27T17:28:26.259ZRunning test: deactivate
1382025-02-27T17:29:20.876ZCompleted test: deactivate
1392025-02-27T17:29:25.883ZRunning hammer
1402025-02-27T17:29:43.991ZRun repair tests
1412025-02-27T17:29:43.995Z/var/tmp/bins/crutest fill -g 31 -q --key 0Md83Np50W5ygafJAJVBph2x9t42z9tuOdLMUgV6x80= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1422025-02-27T17:29:44.053Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.625581991Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"task":"crutest"}
1432025-02-27T17:29:44.056Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.626184926Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"task":"crutest"}
1442025-02-27T17:29:44.056Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.627889374Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"task":"crutest"}
1452025-02-27T17:29:44.056Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.628076692Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238}
1462025-02-27T17:29:44.056Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.628123932Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238}
1472025-02-27T17:29:44.056Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.628146712Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238}
1482025-02-27T17:29:44.056Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.628162622Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238}
1492025-02-27T17:29:44.059Z{"msg":"Crucible 1ebeae9e-0870-41c9-9635-6faeae639d6f has session id: 015108ac-24d7-4f83-b234-e0dd7624837d","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.628768597Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1502025-02-27T17:29:44.059Z{"msg":"Upstairs opts: Upstairs UUID: 1ebeae9e-0870-41c9-9635-6faeae639d6f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.628803927Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1512025-02-27T17:29:44.059Z{"msg":"Crucible stats registered with UUID: 1ebeae9e-0870-41c9-9635-6faeae639d6f","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.628820407Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1522025-02-27T17:29:44.059Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:29:42.628835407Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1532025-02-27T17:29:44.059Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.630901861Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"task":"crutest"}
1542025-02-27T17:29:44.059Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.631004791Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238}
1552025-02-27T17:29:44.059Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.63104115Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238}
1562025-02-27T17:29:44.059Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.6310641Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238}
1572025-02-27T17:29:44.059Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.63108787Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238}
1582025-02-27T17:29:44.059Z{"msg":"Crucible 1ebeae9e-0870-41c9-9635-6faeae639d6f has session id: c4d3b220-c636-43f3-a96c-40e026d8a595","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.631514347Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
1592025-02-27T17:29:44.059Z{"msg":"Upstairs opts: Upstairs UUID: 1ebeae9e-0870-41c9-9635-6faeae639d6f, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.631574266Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
1602025-02-27T17:29:44.063Z{"msg":"Crucible stats registered with UUID: 1ebeae9e-0870-41c9-9635-6faeae639d6f","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.631596926Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
1612025-02-27T17:29:44.063Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:29:42.631628416Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
1622025-02-27T17:29:44.063Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.632362111Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"task":"crutest"}
1632025-02-27T17:29:44.063Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.63240082Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238}
1642025-02-27T17:29:44.063Z{"msg":"1ebeae9e-0870-41c9-9635-6faeae639d6f active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.63247967Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1652025-02-27T17:29:44.064Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.63251163Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"client":"0","":"downstairs","session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1662025-02-27T17:29:44.064Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.632551149Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"client":"0","":"downstairs","session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1672025-02-27T17:29:44.064Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.632584819Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"client":"1","":"downstairs","session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1682025-02-27T17:29:44.064Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.632607169Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"client":"1","":"downstairs","session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1692025-02-27T17:29:44.064Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.632621689Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"client":"2","":"downstairs","session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1702025-02-27T17:29:44.064Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.632655278Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"client":"2","":"downstairs","session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1712025-02-27T17:29:44.064Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.632673338Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1722025-02-27T17:29:44.064Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.632695698Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1732025-02-27T17:29:44.064Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.632719588Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1742025-02-27T17:29:44.064Z{"msg":"ds_connection connected from Ok(127.0.0.1:57273)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.632796357Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1752025-02-27T17:29:44.064Z{"msg":"ds_connection connected from Ok(127.0.0.1:42505)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.632853837Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1762025-02-27T17:29:44.064Z{"msg":"ds_connection connected from Ok(127.0.0.1:53893)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.632993736Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1772025-02-27T17:29:44.064Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.633511742Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"client":"1","":"downstairs","session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1782025-02-27T17:29:44.064Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.633565442Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"client":"2","":"downstairs","session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1792025-02-27T17:29:44.064Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.633591231Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"client":"0","":"downstairs","session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1802025-02-27T17:29:44.064Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.634175627Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"client":"1","":"downstairs","session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1812025-02-27T17:29:44.064Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.634214157Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"client":"2","":"downstairs","session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1822025-02-27T17:29:44.064Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.634230787Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"client":"0","":"downstairs","session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1832025-02-27T17:29:44.064Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.634608114Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1842025-02-27T17:29:44.065Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.634643364Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1852025-02-27T17:29:44.065Z{"msg":"[0]R flush_numbers: [4002, 4002, 4002, 4002, 4002]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.634658523Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"downstairs","":"downstairs","session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1862025-02-27T17:29:44.065Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.634680963Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"downstairs","":"downstairs","session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1872025-02-27T17:29:44.065Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.634695173Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"downstairs","":"downstairs","session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1882025-02-27T17:29:44.065Z{"msg":"[1]R flush_numbers: [4002, 4002, 4002, 4002, 4002]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.634709993Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"downstairs","":"downstairs","session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1892025-02-27T17:29:44.065Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.634730423Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"downstairs","":"downstairs","session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1902025-02-27T17:29:44.065Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.634743963Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"downstairs","":"downstairs","session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1912025-02-27T17:29:44.065Z{"msg":"[2]R flush_numbers: [4002, 4002, 4002, 4002, 4002]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.634764143Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"downstairs","":"downstairs","session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1922025-02-27T17:29:44.065Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.634777183Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"downstairs","":"downstairs","session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1932025-02-27T17:29:44.065Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.634797253Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"downstairs","":"downstairs","session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1942025-02-27T17:29:44.065Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.634810322Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"downstairs","":"downstairs","session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1952025-02-27T17:29:44.065Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.634823872Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"downstairs","":"downstairs","session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1962025-02-27T17:29:44.065Z{"msg":"Next flush: 4003","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.634836692Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"downstairs","":"downstairs","session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1972025-02-27T17:29:44.065Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.634849522Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"downstairs","":"downstairs","session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1982025-02-27T17:29:44.065Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.634874532Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
1992025-02-27T17:29:44.065Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.634886822Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"client":"0","":"downstairs","session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
2002025-02-27T17:29:44.065Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.634907572Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"client":"1","":"downstairs","session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
2012025-02-27T17:29:44.065Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.634942921Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"client":"2","":"downstairs","session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
2022025-02-27T17:29:44.066Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.634981121Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
2032025-02-27T17:29:44.066Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.635041441Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
2042025-02-27T17:29:44.066Z{"msg":"1ebeae9e-0870-41c9-9635-6faeae639d6f is now active with session: 015108ac-24d7-4f83-b234-e0dd7624837d","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.63507696Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
2052025-02-27T17:29:44.066Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.635209319Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"session_id":"015108ac-24d7-4f83-b234-e0dd7624837d"}
2062025-02-27T17:29:44.066Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.635238639Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238}
2072025-02-27T17:29:44.066Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.635285969Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238}
2082025-02-27T17:29:44.066Z{"msg":"1ebeae9e-0870-41c9-9635-6faeae639d6f active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.635326099Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2092025-02-27T17:29:44.066Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.635352558Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"client":"0","":"downstairs","session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2102025-02-27T17:29:44.069Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.635380178Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"client":"0","":"downstairs","session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2112025-02-27T17:29:44.069Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.635417768Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"client":"1","":"downstairs","session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2122025-02-27T17:29:44.069Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.635446048Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"client":"1","":"downstairs","session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2132025-02-27T17:29:44.069Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.635474528Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"client":"2","":"downstairs","session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2142025-02-27T17:29:44.069Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.635516197Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"client":"2","":"downstairs","session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2152025-02-27T17:29:44.069Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.635545777Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2162025-02-27T17:29:44.069Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.635572997Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2172025-02-27T17:29:44.069Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.635598747Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2182025-02-27T17:29:44.069Z{"msg":"ds_connection connected from Ok(127.0.0.1:53024)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.635646376Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2192025-02-27T17:29:44.069Z{"msg":"ds_connection connected from Ok(127.0.0.1:44072)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.635677216Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2202025-02-27T17:29:44.069Z{"msg":"ds_connection connected from Ok(127.0.0.1:41769)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.635704516Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2212025-02-27T17:29:44.069Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.635749855Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"client":"1","":"downstairs","session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2222025-02-27T17:29:44.069Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.635767355Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"client":"2","":"downstairs","session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2232025-02-27T17:29:44.069Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.635789155Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"client":"0","":"downstairs","session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2242025-02-27T17:29:44.069Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.636183452Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"client":"1","":"downstairs","session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2252025-02-27T17:29:44.069Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.636226922Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"client":"2","":"downstairs","session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2262025-02-27T17:29:44.069Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.636254022Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"client":"0","":"downstairs","session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2272025-02-27T17:29:44.069Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.636571769Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2282025-02-27T17:29:44.069Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.636675988Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2292025-02-27T17:29:44.070Z{"msg":"[0]R flush_numbers: [5320, 5319, 5305, 5320, 5320]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.636737368Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"downstairs","":"downstairs","session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2302025-02-27T17:29:44.070Z{"msg":"[0]R generation: [14, 13, 9, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.636754408Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"downstairs","":"downstairs","session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2312025-02-27T17:29:44.070Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.636777068Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"downstairs","":"downstairs","session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2322025-02-27T17:29:44.070Z{"msg":"[1]R flush_numbers: [5320, 5319, 5305, 5320, 5320]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.636808218Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"downstairs","":"downstairs","session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2332025-02-27T17:29:44.070Z{"msg":"[1]R generation: [14, 13, 9, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.636829427Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"downstairs","":"downstairs","session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2342025-02-27T17:29:44.070ZWait for a query_work_queue command to finish before sending IO
2352025-02-27T17:29:44.070Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.636864377Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"downstairs","":"downstairs","session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2362025-02-27T17:29:44.070Z{"msg":"[2]R flush_numbers: [5320, 5319, 5305, 5320, 5320]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.636886207Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"downstairs","":"downstairs","session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2372025-02-27T17:29:44.070Z{"msg":"[2]R generation: [14, 13, 9, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.636899907Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"downstairs","":"downstairs","session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2382025-02-27T17:29:44.070Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.636919857Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"downstairs","":"downstairs","session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2392025-02-27T17:29:44.070Z{"msg":"Max found gen is 15","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.636933007Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"downstairs","":"downstairs","session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2402025-02-27T17:29:44.070Z{"msg":"Generation requested: 31 >= found:15","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.636952257Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"downstairs","":"downstairs","session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2412025-02-27T17:29:44.070Z{"msg":"Next flush: 5321","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.636965077Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"downstairs","":"downstairs","session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2422025-02-27T17:29:44.070Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.636984966Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"":"downstairs","":"downstairs","session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2432025-02-27T17:29:44.070Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.636998066Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2442025-02-27T17:29:44.070Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.637016356Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"client":"0","":"downstairs","session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2452025-02-27T17:29:44.070Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.637036406Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"client":"1","":"downstairs","session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2462025-02-27T17:29:44.070Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.637049346Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"client":"2","":"downstairs","session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2472025-02-27T17:29:44.070Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.637068816Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2482025-02-27T17:29:44.070Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.637081085Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2492025-02-27T17:29:44.070Z{"msg":"1ebeae9e-0870-41c9-9635-6faeae639d6f is now active with session: c4d3b220-c636-43f3-a96c-40e026d8a595","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.637100685Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2502025-02-27T17:29:44.070Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.637112545Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238,"session_id":"c4d3b220-c636-43f3-a96c-40e026d8a595"}
2512025-02-27T17:29:44.070Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:42.637166915Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1238}
2522025-02-27T17:29:44.070ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
2532025-02-27T17:29:44.070ZFill test
2542025-02-27T17:29:44.107ZRead and Verify all blocks (0..100 range:false)
2552025-02-27T17:29:44.186ZWrote out file "/var/tmp/test_up-build/verify_file"
2562025-02-27T17:29:44.186ZCLIENT: Tests done. All submitted work has been ACK'd
2572025-02-27T17:29:44.189Z----------------------------------------------------------------
2582025-02-27T17:29:44.189Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2592025-02-27T17:29:44.189ZStates: Active Active Active
2602025-02-27T17:29:44.189ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2612025-02-27T17:29:44.189Z 1002 Acked Read 50 Done Done Done false
2622025-02-27T17:29:44.189Z STATES DS:0 DS:1 DS:2 TOTAL
2632025-02-27T17:29:44.189Z Sent 0 0 0 0
2642025-02-27T17:29:44.189Z Done 1 1 1 3
2652025-02-27T17:29:44.189Z Skipped 0 0 0 0
2662025-02-27T17:29:44.189Z Error 0 0 0 0
2672025-02-27T17:29:44.189ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2682025-02-27T17:29:44.189ZDownstairs last five completed: 1001 1000
2692025-02-27T17:29:44.189ZUpstairs last five completed: 1002 1001 1000
2702025-02-27T17:29:44.189Z----------------------------------------------------------------
2712025-02-27T17:29:44.189Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2722025-02-27T17:29:44.189ZStates: Active Active Active
2732025-02-27T17:29:44.189ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2742025-02-27T17:29:44.189Z 1002 Acked Read 50 Done Done Done false
2752025-02-27T17:29:44.189Z STATES DS:0 DS:1 DS:2 TOTAL
2762025-02-27T17:29:44.189Z Sent 0 0 0 0
2772025-02-27T17:29:44.189Z Done 1 1 1 3
2782025-02-27T17:29:44.189Z Skipped 0 0 0 0
2792025-02-27T17:29:44.189Z Error 0 0 0 0
2802025-02-27T17:29:44.189ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2812025-02-27T17:29:44.189ZDownstairs last five completed: 1001 1000
2822025-02-27T17:29:44.189ZUpstairs last five completed: 1002 1001 1000
2832025-02-27T17:29:44.189ZCLIENT: Up:0 ds:2 act:6
2842025-02-27T17:29:48.188Z----------------------------------------------------------------
2852025-02-27T17:29:48.189Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2862025-02-27T17:29:48.189ZDownstairs last five completed: 1003 1002 1001 1000
2872025-02-27T17:29:48.189ZUpstairs last five completed: 1003 1002 1001 1000
2882025-02-27T17:29:48.192Z----------------------------------------------------------------
2892025-02-27T17:29:48.192Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2902025-02-27T17:29:48.192ZDownstairs last five completed: 1003 1002 1001 1000
2912025-02-27T17:29:48.192ZUpstairs last five completed: 1003 1002 1001 1000
2922025-02-27T17:29:48.192ZCLIENT: Up:0 ds:0 act:6
2932025-02-27T17:29:48.192ZCLIENT: All crucible jobs finished, exiting program
2942025-02-27T17:29:48.200ZRepair setup passed
2952025-02-27T17:29:48.203ZCopy the region for /var/tmp/test_up-build/8830
2962025-02-27T17:29:48.206Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2972025-02-27T17:29:48.210Z/var/tmp/bins/crutest fill -g 32 -q --key 0Md83Np50W5ygafJAJVBph2x9t42z9tuOdLMUgV6x80= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2982025-02-27T17:29:48.264Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.836730499Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"task":"crutest"}
2992025-02-27T17:29:48.267Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.837365174Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"task":"crutest"}
3002025-02-27T17:29:48.267Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.839167041Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"task":"crutest"}
3012025-02-27T17:29:48.267Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.83935214Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244}
3022025-02-27T17:29:48.267Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.839407689Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244}
3032025-02-27T17:29:48.267Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.839434449Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244}
3042025-02-27T17:29:48.267Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.839453009Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244}
3052025-02-27T17:29:48.270Z{"msg":"Crucible 7bfc629b-987a-4c2a-8934-1b652eb3e666 has session id: 07a77253-9d79-4269-b0fd-93f57f5d94cb","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.840065194Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3062025-02-27T17:29:48.270Z{"msg":"Upstairs opts: Upstairs UUID: 7bfc629b-987a-4c2a-8934-1b652eb3e666, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.840105574Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3072025-02-27T17:29:48.270Z{"msg":"Crucible stats registered with UUID: 7bfc629b-987a-4c2a-8934-1b652eb3e666","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.840121354Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3082025-02-27T17:29:48.270Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:29:46.840139504Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3092025-02-27T17:29:48.270Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.842199299Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"task":"crutest"}
3102025-02-27T17:29:48.270Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.842287828Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244}
3112025-02-27T17:29:48.270Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.842324278Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244}
3122025-02-27T17:29:48.271Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.842356147Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244}
3132025-02-27T17:29:48.271Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.842378617Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244}
3142025-02-27T17:29:48.271Z{"msg":"Crucible 7bfc629b-987a-4c2a-8934-1b652eb3e666 has session id: 98b7b512-ed4f-4e69-95b1-13602194c26d","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.842785034Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3152025-02-27T17:29:48.271Z{"msg":"Upstairs opts: Upstairs UUID: 7bfc629b-987a-4c2a-8934-1b652eb3e666, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.842830374Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3162025-02-27T17:29:48.271Z{"msg":"Crucible stats registered with UUID: 7bfc629b-987a-4c2a-8934-1b652eb3e666","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.842850284Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3172025-02-27T17:29:48.271Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:29:46.842878983Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3182025-02-27T17:29:48.274Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.843491899Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"task":"crutest"}
3192025-02-27T17:29:48.274Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.843568849Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244}
3202025-02-27T17:29:48.274Z{"msg":"7bfc629b-987a-4c2a-8934-1b652eb3e666 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.843586888Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3212025-02-27T17:29:48.274Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.843601128Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"client":"0","":"downstairs","session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3222025-02-27T17:29:48.274Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.843645928Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"client":"0","":"downstairs","session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3232025-02-27T17:29:48.274Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.843672148Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"client":"1","":"downstairs","session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3242025-02-27T17:29:48.274Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.843711907Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"client":"1","":"downstairs","session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3252025-02-27T17:29:48.274Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.843738697Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"client":"2","":"downstairs","session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3262025-02-27T17:29:48.274Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.843775217Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"client":"2","":"downstairs","session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3272025-02-27T17:29:48.275Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.843808977Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3282025-02-27T17:29:48.275Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.843835507Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3292025-02-27T17:29:48.275Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.843880166Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3302025-02-27T17:29:48.275Z{"msg":"ds_connection connected from Ok(127.0.0.1:45115)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.843905016Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3312025-02-27T17:29:48.275Z{"msg":"ds_connection connected from Ok(127.0.0.1:58196)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.843938276Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3322025-02-27T17:29:48.275Z{"msg":"ds_connection connected from Ok(127.0.0.1:39993)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.844085245Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3332025-02-27T17:29:48.275Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.84466051Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"client":"2","":"downstairs","session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3342025-02-27T17:29:48.275Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.8447049Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"client":"1","":"downstairs","session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3352025-02-27T17:29:48.275Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.84472738Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"client":"0","":"downstairs","session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3362025-02-27T17:29:48.275Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.845282646Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"client":"1","":"downstairs","session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3372025-02-27T17:29:48.275Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.845322856Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"client":"0","":"downstairs","session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3382025-02-27T17:29:48.275Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.845429375Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"client":"2","":"downstairs","session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3392025-02-27T17:29:48.275Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.845740862Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3402025-02-27T17:29:48.275Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.845785772Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3412025-02-27T17:29:48.276Z{"msg":"[0]R flush_numbers: [4003, 4003, 4003, 4003, 4003]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.845809202Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"downstairs","":"downstairs","session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3422025-02-27T17:29:48.276Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.845844332Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"downstairs","":"downstairs","session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3432025-02-27T17:29:48.276Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.845878531Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"downstairs","":"downstairs","session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3442025-02-27T17:29:48.276Z{"msg":"[1]R flush_numbers: [4003, 4003, 4003, 4003, 4003]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.845911291Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"downstairs","":"downstairs","session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3452025-02-27T17:29:48.276Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.845944661Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"downstairs","":"downstairs","session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3462025-02-27T17:29:48.276Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.845980391Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"downstairs","":"downstairs","session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3472025-02-27T17:29:48.276Z{"msg":"[2]R flush_numbers: [4003, 4003, 4003, 4003, 4003]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.84602054Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"downstairs","":"downstairs","session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3482025-02-27T17:29:48.276Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.84606187Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"downstairs","":"downstairs","session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3492025-02-27T17:29:48.276Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.8461053Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"downstairs","":"downstairs","session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3502025-02-27T17:29:48.276Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.846160099Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"downstairs","":"downstairs","session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3512025-02-27T17:29:48.276Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.846185109Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"downstairs","":"downstairs","session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3522025-02-27T17:29:48.276Z{"msg":"Next flush: 4004","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.846220429Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"downstairs","":"downstairs","session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3532025-02-27T17:29:48.276Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.846246238Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"downstairs","":"downstairs","session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3542025-02-27T17:29:48.276Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.846279008Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3552025-02-27T17:29:48.279Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.846406397Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"client":"0","":"downstairs","session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3562025-02-27T17:29:48.279Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.846444577Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"client":"1","":"downstairs","session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3572025-02-27T17:29:48.279Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.846470067Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"client":"2","":"downstairs","session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3582025-02-27T17:29:48.279Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.846507267Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3592025-02-27T17:29:48.279Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.846533607Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3602025-02-27T17:29:48.280Z{"msg":"7bfc629b-987a-4c2a-8934-1b652eb3e666 is now active with session: 07a77253-9d79-4269-b0fd-93f57f5d94cb","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.846580026Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3612025-02-27T17:29:48.280Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.846616226Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"session_id":"07a77253-9d79-4269-b0fd-93f57f5d94cb"}
3622025-02-27T17:29:48.280Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.846646326Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244}
3632025-02-27T17:29:48.280Z{"msg":"7bfc629b-987a-4c2a-8934-1b652eb3e666 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.846664075Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3642025-02-27T17:29:48.280Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.846701305Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"client":"0","":"downstairs","session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3652025-02-27T17:29:48.280Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.846723135Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244}
3662025-02-27T17:29:48.280Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.846748335Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"client":"0","":"downstairs","session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3672025-02-27T17:29:48.280Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.846774575Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"client":"1","":"downstairs","session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3682025-02-27T17:29:48.280Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.846797034Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"client":"1","":"downstairs","session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3692025-02-27T17:29:48.280Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.846825494Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"client":"2","":"downstairs","session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3702025-02-27T17:29:48.280Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.846841494Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"client":"2","":"downstairs","session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3712025-02-27T17:29:48.280Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.846873644Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3722025-02-27T17:29:48.280Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.846894874Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3732025-02-27T17:29:48.280Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.846932024Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3742025-02-27T17:29:48.280Z{"msg":"ds_connection connected from Ok(127.0.0.1:50095)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.846949393Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3752025-02-27T17:29:48.280Z{"msg":"ds_connection connected from Ok(127.0.0.1:38094)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.846982103Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3762025-02-27T17:29:48.280Z{"msg":"ds_connection connected from Ok(127.0.0.1:40609)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.847005623Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3772025-02-27T17:29:48.280Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.847032683Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"client":"1","":"downstairs","session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3782025-02-27T17:29:48.280Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.847058472Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"client":"2","":"downstairs","session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3792025-02-27T17:29:48.280Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.847077452Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"client":"0","":"downstairs","session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3802025-02-27T17:29:48.280Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.84740207Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"client":"1","":"downstairs","session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3812025-02-27T17:29:48.280Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.847557989Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"client":"0","":"downstairs","session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3822025-02-27T17:29:48.280Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.847696608Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"client":"2","":"downstairs","session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3832025-02-27T17:29:48.280Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.847875606Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3842025-02-27T17:29:48.280Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.847920406Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3852025-02-27T17:29:48.280Z{"msg":"[0]R flush_numbers: [5321, 5321, 5321, 5321, 5321]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.848014125Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"downstairs","":"downstairs","session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3862025-02-27T17:29:48.280Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.848050235Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"downstairs","":"downstairs","session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3872025-02-27T17:29:48.280Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.848064355Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"downstairs","":"downstairs","session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3882025-02-27T17:29:48.280Z{"msg":"[1]R flush_numbers: [5321, 5321, 5321, 5321, 5321]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.848092385Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"downstairs","":"downstairs","session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3892025-02-27T17:29:48.280Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.848107145Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"downstairs","":"downstairs","session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3902025-02-27T17:29:48.280Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.848120525Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"downstairs","":"downstairs","session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3912025-02-27T17:29:48.280Z{"msg":"[2]R flush_numbers: [5321, 5321, 5321, 5321, 5321]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.848141144Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"downstairs","":"downstairs","session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3922025-02-27T17:29:48.280Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.848155064Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"downstairs","":"downstairs","session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3932025-02-27T17:29:48.282Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.848174634Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"downstairs","":"downstairs","session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3942025-02-27T17:29:48.282Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.848187624Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"downstairs","":"downstairs","session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3952025-02-27T17:29:48.282Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.848200394Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"downstairs","":"downstairs","session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3962025-02-27T17:29:48.282ZWait for a query_work_queue command to finish before sending IO
3972025-02-27T17:29:48.282Z{"msg":"Next flush: 5322","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.848237434Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"downstairs","":"downstairs","session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3982025-02-27T17:29:48.282Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.848251344Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"":"downstairs","":"downstairs","session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
3992025-02-27T17:29:48.282Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.848273723Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
4002025-02-27T17:29:48.282Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.848305223Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"client":"0","":"downstairs","session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
4012025-02-27T17:29:48.282Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.848341743Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"client":"1","":"downstairs","session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
4022025-02-27T17:29:48.282Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.848369273Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"client":"2","":"downstairs","session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
4032025-02-27T17:29:48.282Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.848404722Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
4042025-02-27T17:29:48.282Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.848430572Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
4052025-02-27T17:29:48.282Z{"msg":"7bfc629b-987a-4c2a-8934-1b652eb3e666 is now active with session: 98b7b512-ed4f-4e69-95b1-13602194c26d","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.848466632Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
4062025-02-27T17:29:48.282Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.848500962Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244,"session_id":"98b7b512-ed4f-4e69-95b1-13602194c26d"}
4072025-02-27T17:29:48.282Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:46.848527542Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244}
4082025-02-27T17:29:48.282ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
4092025-02-27T17:29:48.282ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4102025-02-27T17:29:48.282ZFill test
4112025-02-27T17:29:48.318ZRead and Verify all blocks (0..100 range:false)
4122025-02-27T17:29:48.402ZWrote out file "/var/tmp/test_up-build/verify_file"
4132025-02-27T17:29:48.402ZCLIENT: Tests done. All submitted work has been ACK'd
4142025-02-27T17:29:48.404Z----------------------------------------------------------------
4152025-02-27T17:29:48.405Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4162025-02-27T17:29:48.405ZStates: Active Active Active
4172025-02-27T17:29:48.405ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4182025-02-27T17:29:48.405Z 1002 Acked Read 50 Done Done Done false
4192025-02-27T17:29:48.405Z STATES DS:0 DS:1 DS:2 TOTAL
4202025-02-27T17:29:48.405Z Sent 0 0 0 0
4212025-02-27T17:29:48.405Z Done 1 1 1 3
4222025-02-27T17:29:48.405Z Skipped 0 0 0 0
4232025-02-27T17:29:48.405Z Error 0 0 0 0
4242025-02-27T17:29:48.405ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4252025-02-27T17:29:48.405ZDownstairs last five completed: 1001 1000
4262025-02-27T17:29:48.405ZUpstairs last five completed: 1002 1001 1000
4272025-02-27T17:29:48.405Z----------------------------------------------------------------
4282025-02-27T17:29:48.405Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4292025-02-27T17:29:48.405ZStates: Active Active Active
4302025-02-27T17:29:48.405ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4312025-02-27T17:29:48.405Z 1002 Acked Read 50 Done Done Done false
4322025-02-27T17:29:48.406Z STATES DS:0 DS:1 DS:2 TOTAL
4332025-02-27T17:29:48.406Z Sent 0 0 0 0
4342025-02-27T17:29:48.406Z Done 1 1 1 3
4352025-02-27T17:29:48.406Z Skipped 0 0 0 0
4362025-02-27T17:29:48.406Z Error 0 0 0 0
4372025-02-27T17:29:48.406ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4382025-02-27T17:29:48.406ZDownstairs last five completed: 1001 1000
4392025-02-27T17:29:48.406ZUpstairs last five completed: 1002 1001 1000
4402025-02-27T17:29:48.406ZCLIENT: Up:0 ds:2 act:6
4412025-02-27T17:29:52.404Z----------------------------------------------------------------
4422025-02-27T17:29:52.404Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4432025-02-27T17:29:52.404ZDownstairs last five completed: 1003 1002 1001 1000
4442025-02-27T17:29:52.404ZUpstairs last five completed: 1003 1002 1001 1000
4452025-02-27T17:29:52.404Z----------------------------------------------------------------
4462025-02-27T17:29:52.404Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4472025-02-27T17:29:52.404ZDownstairs last five completed: 1003 1002 1001 1000
4482025-02-27T17:29:52.404ZUpstairs last five completed: 1003 1002 1001 1000
4492025-02-27T17:29:52.406ZCLIENT: Up:0 ds:0 act:6
4502025-02-27T17:29:52.407ZCLIENT: All crucible jobs finished, exiting program
4512025-02-27T17:29:52.407Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:29:50.976616865Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1244}
4522025-02-27T17:29:52.411ZRepair part 1 passed
4532025-02-27T17:29:52.413Z
4542025-02-27T17:29:52.413ZKill the current downstairs
4552025-02-27T17:29:52.577ZDownstairs 2 stopped
4562025-02-27T17:29:52.577Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
4572025-02-27T17:29:52.580ZNow put back the original so we have a mismatch
4582025-02-27T17:29:52.580Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
4592025-02-27T17:29:52.582ZRestart downstairs with old directory
4602025-02-27T17:29:52.634Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4612025-02-27T17:29:52.650ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4622025-02-27T17:29:52.654Z 0 000-009 32 32 31 4004 4004 4003 F F F <---
4632025-02-27T17:29:52.654Z 1 010-019 32 32 31 4004 4004 4003 F F F <---
4642025-02-27T17:29:52.654Z 2 020-029 32 32 31 4004 4004 4003 F F F <---
4652025-02-27T17:29:52.654Z 3 030-039 32 32 31 4004 4004 4003 F F F <---
4662025-02-27T17:29:52.654Z 4 040-049 32 32 31 4004 4004 4003 F F F <---
4672025-02-27T17:29:52.654ZMax gen: 32, Max flush: 4004
4682025-02-27T17:29:52.654ZError: Difference in extent metadata found!
4692025-02-27T17:29:52.657Zdump test found error as expected
4702025-02-27T17:29:52.657Z
4712025-02-27T17:29:52.659Z
4722025-02-27T17:29:52.659Z/var/tmp/bins/crutest verify --range -g 33 -q --key 0Md83Np50W5ygafJAJVBph2x9t42z9tuOdLMUgV6x80= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
4732025-02-27T17:29:52.710Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.283007215Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"task":"crutest"}
4742025-02-27T17:29:52.713Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.283542891Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"task":"crutest"}
4752025-02-27T17:29:52.713Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.285176679Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"task":"crutest"}
4762025-02-27T17:29:52.713Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.285357527Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252}
4772025-02-27T17:29:52.713Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.285409877Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252}
4782025-02-27T17:29:52.713Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.285437647Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252}
4792025-02-27T17:29:52.713Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.285455827Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252}
4802025-02-27T17:29:52.716Z{"msg":"Crucible a828ee63-e2cf-4d79-a3a4-6ccda255a3b8 has session id: ecd6ad26-7b90-4b87-89b4-d11d415e77f0","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.286039202Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
4812025-02-27T17:29:52.716Z{"msg":"Upstairs opts: Upstairs UUID: a828ee63-e2cf-4d79-a3a4-6ccda255a3b8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.286091302Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
4822025-02-27T17:29:52.716Z{"msg":"Crucible stats registered with UUID: a828ee63-e2cf-4d79-a3a4-6ccda255a3b8","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.286110322Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
4832025-02-27T17:29:52.717Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:29:51.286127332Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
4842025-02-27T17:29:52.717Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.288053978Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"task":"crutest"}
4852025-02-27T17:29:52.717Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.288139977Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252}
4862025-02-27T17:29:52.717Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.288178137Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252}
4872025-02-27T17:29:52.717Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.288221586Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252}
4882025-02-27T17:29:52.717Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.288238706Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252}
4892025-02-27T17:29:52.717Z{"msg":"Crucible a828ee63-e2cf-4d79-a3a4-6ccda255a3b8 has session id: 0a75d472-9003-458b-b44a-1caf1eba2ac1","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.288582383Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
4902025-02-27T17:29:52.717Z{"msg":"Upstairs opts: Upstairs UUID: a828ee63-e2cf-4d79-a3a4-6ccda255a3b8, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.288628843Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
4912025-02-27T17:29:52.718Z{"msg":"Crucible stats registered with UUID: a828ee63-e2cf-4d79-a3a4-6ccda255a3b8","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.288655343Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
4922025-02-27T17:29:52.718Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:29:51.288685933Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
4932025-02-27T17:29:52.720Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.289259569Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"task":"crutest"}
4942025-02-27T17:29:52.720Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.289335738Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252}
4952025-02-27T17:29:52.720Z{"msg":"a828ee63-e2cf-4d79-a3a4-6ccda255a3b8 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.289373898Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
4962025-02-27T17:29:52.721Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.289398048Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"0","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
4972025-02-27T17:29:52.721Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.289455057Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"0","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
4982025-02-27T17:29:52.721Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.289478217Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"1","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
4992025-02-27T17:29:52.721Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.289509517Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"1","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5002025-02-27T17:29:52.721Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.289531647Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"2","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5012025-02-27T17:29:52.721Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.289547236Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"2","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5022025-02-27T17:29:52.721Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.289575276Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5032025-02-27T17:29:52.721Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.289598516Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5042025-02-27T17:29:52.721Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.289632106Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5052025-02-27T17:29:52.721Z{"msg":"ds_connection connected from Ok(127.0.0.1:43791)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.289661375Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5062025-02-27T17:29:52.721Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:29:51.289701205Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5072025-02-27T17:29:52.721Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:29:51.289732005Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5082025-02-27T17:29:52.721Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:29:51.289771705Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5092025-02-27T17:29:52.721Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.289965823Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5102025-02-27T17:29:52.721Z{"msg":"ds_connection connected from Ok(127.0.0.1:36757)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.290039183Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5112025-02-27T17:29:52.721Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.290226042Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"1","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5122025-02-27T17:29:52.721Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.29037209Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"0","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5132025-02-27T17:29:52.721Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.290749318Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"1","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5142025-02-27T17:29:52.721Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.290887206Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"0","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5152025-02-27T17:29:52.721Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.291149795Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5162025-02-27T17:29:52.721Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:51.291244474Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5172025-02-27T17:30:02.720Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.291239092Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5182025-02-27T17:30:02.724Z{"msg":"ds_connection connected from Ok(127.0.0.1:42567)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.29149816Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5192025-02-27T17:30:02.724Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.292482013Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"2","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5202025-02-27T17:30:02.724Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.292996659Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"2","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5212025-02-27T17:30:02.724Z{"msg":"[0]R flush_numbers: [4004, 4004, 4004, 4004, 4004]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.293408535Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"downstairs","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5222025-02-27T17:30:02.724Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.293452995Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"downstairs","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5232025-02-27T17:30:02.724Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.293484805Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"downstairs","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5242025-02-27T17:30:02.724Z{"msg":"[1]R flush_numbers: [4004, 4004, 4004, 4004, 4004]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.293522945Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"downstairs","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5252025-02-27T17:30:02.725Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.293553614Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"downstairs","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5262025-02-27T17:30:02.725Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.293588364Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"downstairs","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5272025-02-27T17:30:02.725Z{"msg":"[2]R flush_numbers: [4003, 4003, 4003, 4003, 4003]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.293618264Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"downstairs","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5282025-02-27T17:30:02.725Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.293651744Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"downstairs","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5292025-02-27T17:30:02.725Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.293681284Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"downstairs","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5302025-02-27T17:30:02.725Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.293714603Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"downstairs","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5312025-02-27T17:30:02.725Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.293742793Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"downstairs","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5322025-02-27T17:30:02.725Z{"msg":"Next flush: 4005","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.293776313Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"downstairs","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5332025-02-27T17:30:02.725Z{"msg":"extent 0 needs reconciliation: ExtentMetadata { gen: 32, flush: 4004, dirty: false } ExtentMetadata { gen: 32, flush: 4004, dirty: false } ExtentMetadata { gen: 31, flush: 4003, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.293810063Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5342025-02-27T17:30:02.725Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 4004, dirty: false }, ExtentMetadata { gen: 32, flush: 4004, dirty: false }, ExtentMetadata { gen: 31, flush: 4003, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.293847972Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5352025-02-27T17:30:02.725Z{"msg":"found dest for source 0 for extent at index 0 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.293879572Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5362025-02-27T17:30:02.725Z{"msg":"extent 1 needs reconciliation: ExtentMetadata { gen: 32, flush: 4004, dirty: false } ExtentMetadata { gen: 32, flush: 4004, dirty: false } ExtentMetadata { gen: 31, flush: 4003, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.293924512Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5372025-02-27T17:30:02.725Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 4004, dirty: false }, ExtentMetadata { gen: 32, flush: 4004, dirty: false }, ExtentMetadata { gen: 31, flush: 4003, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.293961151Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5382025-02-27T17:30:02.725Z{"msg":"found dest for source 0 for extent at index 1 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.293995071Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5392025-02-27T17:30:02.725Z{"msg":"extent 2 needs reconciliation: ExtentMetadata { gen: 32, flush: 4004, dirty: false } ExtentMetadata { gen: 32, flush: 4004, dirty: false } ExtentMetadata { gen: 31, flush: 4003, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.294032481Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5402025-02-27T17:30:02.725Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 4004, dirty: false }, ExtentMetadata { gen: 32, flush: 4004, dirty: false }, ExtentMetadata { gen: 31, flush: 4003, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.294089021Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5412025-02-27T17:30:02.725Z{"msg":"found dest for source 0 for extent at index 2 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.29414428Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5422025-02-27T17:30:02.725Z{"msg":"extent 3 needs reconciliation: ExtentMetadata { gen: 32, flush: 4004, dirty: false } ExtentMetadata { gen: 32, flush: 4004, dirty: false } ExtentMetadata { gen: 31, flush: 4003, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.29418443Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5432025-02-27T17:30:02.725Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 4004, dirty: false }, ExtentMetadata { gen: 32, flush: 4004, dirty: false }, ExtentMetadata { gen: 31, flush: 4003, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.294218009Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5442025-02-27T17:30:02.725Z{"msg":"found dest for source 0 for extent at index 3 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.294268219Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5452025-02-27T17:30:02.725Z{"msg":"extent 4 needs reconciliation: ExtentMetadata { gen: 32, flush: 4004, dirty: false } ExtentMetadata { gen: 32, flush: 4004, dirty: false } ExtentMetadata { gen: 31, flush: 4003, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.294315579Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5462025-02-27T17:30:02.725Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 4004, dirty: false }, ExtentMetadata { gen: 32, flush: 4004, dirty: false }, ExtentMetadata { gen: 31, flush: 4003, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.294347979Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5472025-02-27T17:30:02.725Z{"msg":"found dest for source 0 for extent at index 4 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.294378418Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5482025-02-27T17:30:02.725Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.294406038Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"0","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5492025-02-27T17:30:02.725Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.294421428Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"1","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5502025-02-27T17:30:02.725Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.294435178Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"2","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5512025-02-27T17:30:02.725Z{"msg":"Full repair list: {ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.294478548Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"downstairs","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5522025-02-27T17:30:02.725Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 4005, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 4005, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 4005, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 4005, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 4005, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.294563327Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"downstairs","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5532025-02-27T17:30:02.725Z{"msg":"starting reconciliation e8502c2b-6f51-4418-8469-da893eed34f9: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.294579907Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"downstairs","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5542025-02-27T17:30:02.725Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.294593407Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"2","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5552025-02-27T17:30:02.731Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.303297092Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"2","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5562025-02-27T17:30:02.735Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.307019865Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"2","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5572025-02-27T17:30:02.738Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.310717957Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"2","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5582025-02-27T17:30:02.742Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.314567429Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"2","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5592025-02-27T17:30:02.745Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.317659206Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"downstairs","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5602025-02-27T17:30:02.745Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.317697726Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"0","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5612025-02-27T17:30:02.748Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.317714775Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"1","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5622025-02-27T17:30:02.748Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.317762745Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"2","":"downstairs","session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5632025-02-27T17:30:02.748Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.317786655Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5642025-02-27T17:30:02.748Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.317817025Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5652025-02-27T17:30:02.748Z{"msg":"a828ee63-e2cf-4d79-a3a4-6ccda255a3b8 is now active with session: ecd6ad26-7b90-4b87-89b4-d11d415e77f0","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.317846774Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"session_id":"ecd6ad26-7b90-4b87-89b4-d11d415e77f0"}
5662025-02-27T17:30:02.748Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.317878354Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252}
5672025-02-27T17:30:02.748Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.317900454Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252}
5682025-02-27T17:30:02.748Z{"msg":"a828ee63-e2cf-4d79-a3a4-6ccda255a3b8 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.317919884Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
5692025-02-27T17:30:02.748Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.317941544Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"0","":"downstairs","session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
5702025-02-27T17:30:02.748Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.317965734Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"0","":"downstairs","session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
5712025-02-27T17:30:02.748Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.317989404Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"1","":"downstairs","session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
5722025-02-27T17:30:02.749Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.318016363Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"1","":"downstairs","session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
5732025-02-27T17:30:02.749Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.318039223Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"2","":"downstairs","session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
5742025-02-27T17:30:02.749Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.318063843Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"2","":"downstairs","session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
5752025-02-27T17:30:02.749Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.318084883Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
5762025-02-27T17:30:02.749Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.318105943Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
5772025-02-27T17:30:02.749Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.318126423Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
5782025-02-27T17:30:02.749Z{"msg":"ds_connection connected from Ok(127.0.0.1:39817)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.318162762Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
5792025-02-27T17:30:02.749Z{"msg":"ds_connection connected from Ok(127.0.0.1:57440)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.318203122Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
5802025-02-27T17:30:02.749Z{"msg":"ds_connection connected from Ok(127.0.0.1:64797)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.318231432Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
5812025-02-27T17:30:02.749Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.31842515Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"0","":"downstairs","session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
5822025-02-27T17:30:02.749Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.318597609Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"1","":"downstairs","session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
5832025-02-27T17:30:02.749Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.318645019Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"2","":"downstairs","session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
5842025-02-27T17:30:02.749Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.319050136Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"0","":"downstairs","session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
5852025-02-27T17:30:02.749Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.319144355Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"1","":"downstairs","session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
5862025-02-27T17:30:02.749Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.319190004Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"2","":"downstairs","session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
5872025-02-27T17:30:02.749Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.319342954Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
5882025-02-27T17:30:02.750Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.319483752Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
5892025-02-27T17:30:02.750Z{"msg":"[0]R flush_numbers: [5322, 5322, 5322, 5322, 5322]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.319572742Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"downstairs","":"downstairs","session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
5902025-02-27T17:30:02.750Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.319600241Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"downstairs","":"downstairs","session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
5912025-02-27T17:30:02.750Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.319615572Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"downstairs","":"downstairs","session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
5922025-02-27T17:30:02.750Z{"msg":"[1]R flush_numbers: [5322, 5322, 5322, 5322, 5322]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.319642751Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"downstairs","":"downstairs","session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
5932025-02-27T17:30:02.750Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.319670391Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"downstairs","":"downstairs","session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
5942025-02-27T17:30:02.750Z{"msg"Wait for a query_work_queue command to finish before sending IO
5952025-02-27T17:30:02.750Z:"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.319694871Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"downstairs","":"downstairs","session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
5962025-02-27T17:30:02.750Z{"msg":"[2]R flush_numbers: [5322, 5322, 5322, 5322, 5322]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.319719891Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"downstairs","":"downstairs","session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
5972025-02-27T17:30:02.750Z{"msg":"[2]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.319745831Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"downstairs","":"downstairs","session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
5982025-02-27T17:30:02.750Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.31981177Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"downstairs","":"downstairs","session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
5992025-02-27T17:30:02.750Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.31984189Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"downstairs","":"downstairs","session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
6002025-02-27T17:30:02.750Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.319880919Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"downstairs","":"downstairs","session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
6012025-02-27T17:30:02.750Z{"msg":"Next flush: 5323","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.319920059Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"downstairs","":"downstairs","session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
6022025-02-27T17:30:02.750Z{"msg":"All extents match","v":0,"name":"crucible","level":30Disk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
6032025-02-27T17:30:02.750Z,"time":"2025-02-27T17:30:01.319960479Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"":"downstairs","":"downstairs","session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
6042025-02-27T17:30:02.750Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.320002518Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
6052025-02-27T17:30:02.750Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.320033928Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"0","":"downstairs","session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
6062025-02-27T17:30:02.750Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.320048958Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"1","":"downstairs","session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
6072025-02-27T17:30:02.750Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.320070468Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"client":"2","":"downstairs","session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
6082025-02-27T17:30:02.750Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.320084638Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
6092025-02-27T17:30:02.750Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.320105058Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":Loading write count information from file "/var/tmp/test_up-build/verify_file"
6102025-02-27T17:30:02.751Z1252,"session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
6112025-02-27T17:30:02.751Z{"msg":"a828ee63-e2cf-4d79-a3a4-6ccda255a3b8 is now active with session: 0a75d472-9003-458b-b44a-1caf1eba2ac1","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.320124628Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
6122025-02-27T17:30:02.751ZRead and Verify all blocks (0..100 range:true)
6132025-02-27T17:30:02.751Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.320146048Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252,"session_id":"0a75d472-9003-458b-b44a-1caf1eba2ac1"}
6142025-02-27T17:30:02.751Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:01.320158597Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252}
6152025-02-27T17:30:02.846ZWrote out file "/var/tmp/test_up-build/verify_file"
6162025-02-27T17:30:02.846ZVerify test completed
6172025-02-27T17:30:02.848ZWrote out file "/var/tmp/test_up-build/verify_file"
6182025-02-27T17:30:02.848ZCLIENT: Tests done. All submitted work has been ACK'd
6192025-02-27T17:30:02.848Z----------------------------------------------------------------
6202025-02-27T17:30:02.848Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6212025-02-27T17:30:02.849ZStates: Active Active Active
6222025-02-27T17:30:02.849ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6232025-02-27T17:30:02.849Z 1000 Acked Read 50 Done Done Done false
6242025-02-27T17:30:02.849Z STATES DS:0 DS:1 DS:2 TOTAL
6252025-02-27T17:30:02.849Z Sent 0 0 0 0
6262025-02-27T17:30:02.849Z Done 1 1 1 3
6272025-02-27T17:30:02.849Z Skipped 0 0 0 0
6282025-02-27T17:30:02.849Z Error 0 0 0 0
6292025-02-27T17:30:02.849ZLast Flush: None None None
6302025-02-27T17:30:02.849ZDownstairs last five completed:
6312025-02-27T17:30:02.849ZUpstairs last five completed: 1000
6322025-02-27T17:30:02.849Z----------------------------------------------------------------
6332025-02-27T17:30:02.849Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6342025-02-27T17:30:02.849ZStates: Active Active Active
6352025-02-27T17:30:02.849ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6362025-02-27T17:30:02.849Z 1000 Acked Read 50 Done Done Done false
6372025-02-27T17:30:02.849Z STATES DS:0 DS:1 DS:2 TOTAL
6382025-02-27T17:30:02.849Z Sent 0 0 0 0
6392025-02-27T17:30:02.849Z Done 1 1 1 3
6402025-02-27T17:30:02.849Z Skipped 0 0 0 0
6412025-02-27T17:30:02.849Z Error 0 0 0 0
6422025-02-27T17:30:02.849ZLast Flush: None None None
6432025-02-27T17:30:02.849ZDownstairs last five completed:
6442025-02-27T17:30:02.849ZUpstairs last five completed: 1000
6452025-02-27T17:30:02.849ZCLIENT: Up:0 ds:2 act:6
6462025-02-27T17:30:06.847Z----------------------------------------------------------------
6472025-02-27T17:30:06.847Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6482025-02-27T17:30:06.847ZDownstairs last five completed: 1001 1000
6492025-02-27T17:30:06.847ZUpstairs last five completed: 1001 1000
6502025-02-27T17:30:06.847Z----------------------------------------------------------------
6512025-02-27T17:30:06.847Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6522025-02-27T17:30:06.847ZDownstairs last five completed: 1001 1000
6532025-02-27T17:30:06.847ZUpstairs last five completed: 1001 1000
6542025-02-27T17:30:06.850ZCLIENT: Up:0 ds:0 act:6
6552025-02-27T17:30:06.850ZCLIENT: All crucible jobs finished, exiting program
6562025-02-27T17:30:06.850Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:30:05.419434064Z","hostname":"w-01JN46VKXFNF2X2BGRECDC9DHA","pid":1252}
6572025-02-27T17:30:06.854ZRepair part 2 passed
6582025-02-27T17:30:06.857Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
6592025-02-27T17:30:06.872ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6602025-02-27T17:30:06.874Z 0 000-009 32 32 32 4004 4004 4004 F F F
6612025-02-27T17:30:06.874Z 1 010-019 32 32 32 4004 4004 4004 F F F
6622025-02-27T17:30:06.874Z 2 020-029 32 32 32 4004 4004 4004 F F F
6632025-02-27T17:30:06.874Z 3 030-039 32 32 32 4004 4004 4004 F F F
6642025-02-27T17:30:06.875Z 4 040-049 32 32 32 4004 4004 4004 F F F
6652025-02-27T17:30:06.875ZMax gen: 32, Max flush: 4004
6662025-02-27T17:30:06.877Zdump test passed
6672025-02-27T17:30:06.877Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
6682025-02-27T17:30:06.891Z Extent 1
6692025-02-27T17:30:06.894ZGEN 32 32 32
6702025-02-27T17:30:06.894ZFLUSH_ID 4004 4004 4004
6712025-02-27T17:30:06.894ZDIRTY
6722025-02-27T17:30:06.894Z
6732025-02-27T17:30:06.894ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
6742025-02-27T17:30:06.894Z010 A A A A A A
6752025-02-27T17:30:06.897Z011 A A A A A A
6762025-02-27T17:30:06.897Z012 A A A A A A
6772025-02-27T17:30:06.900Z013 A A A A A A
6782025-02-27T17:30:06.900Z014 A A A A A A
6792025-02-27T17:30:06.902Z015 A A A A A A
6802025-02-27T17:30:06.902Z016 A A A A A A
6812025-02-27T17:30:06.905Z017 A A A A A A
6822025-02-27T17:30:06.905Z018 A A A A A A
6832025-02-27T17:30:06.907Z019 A A A A A A
6842025-02-27T17:30:06.910Zdump extent test passed
6852025-02-27T17:30:06.910Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
6862025-02-27T17:30:06.925ZExtent 2 Block in extent 0 Actual block 20
6872025-02-27T17:30:06.928Z
6882025-02-27T17:30:06.928Z DATA SHA256 VER
6892025-02-27T17:30:06.928Z------ ---------------------------------------------------------------- ---
6902025-02-27T17:30:06.928Z 0 8dbfe8e40402679888ef6b490922ab5df1bf2522a92d5de32f36343a4349d270 A
6912025-02-27T17:30:06.928Z 1 8dbfe8e40402679888ef6b490922ab5df1bf2522a92d5de32f36343a4349d270 A
6922025-02-27T17:30:06.928Z 2 8dbfe8e40402679888ef6b490922ab5df1bf2522a92d5de32f36343a4349d270 A
6932025-02-27T17:30:06.928Z
6942025-02-27T17:30:06.928ZNONCES 0 1 2 DIFF
6952025-02-27T17:30:06.928Z------ ------------------------ ------------------------ ------------------------ -----
6962025-02-27T17:30:06.928Z 0 019969cb536ca6d68accf904 019969cb536ca6d68accf904 019969cb536ca6d68accf904
6972025-02-27T17:30:06.928Z
6982025-02-27T17:30:06.928Z TAGS 0 1 2 DIFF
6992025-02-27T17:30:06.928Z------ -------------------------------- -------------------------------- -------------------------------- -----
7002025-02-27T17:30:06.928Z 0 36efd22949c222d8d102b0dff7eec224 36efd22949c222d8d102b0dff7eec224 36efd22949c222d8d102b0dff7eec224
7012025-02-27T17:30:06.929Z
7022025-02-27T17:30:06.929ZHASHES 0 1 2 DIFF
7032025-02-27T17:30:06.929Z------ ---------------- ---------------- ---------------- -----
7042025-02-27T17:30:06.929Z
7052025-02-27T17:30:06.932Zdump block test passed
7062025-02-27T17:30:06.932ZInitial upstairs tests have completed, stopping all downstairs
7072025-02-27T17:30:11.995ZCreating 7 larger downstairs regions
7082025-02-27T17:30:12.174ZStarting 7 downstairs
7092025-02-27T17:30:17.181Zdsc restarted at PID: 1268
7102025-02-27T17:30:17.563ZNow do the replace-reconcile test
7112025-02-27T17:30:17.618ZUsing 8870 for the replacement port
7122025-02-27T17:32:54.933ZCompleted test: replace-reconcile
7132025-02-27T17:32:54.936ZNow do the replace-before-active test
7142025-02-27T17:36:28.981ZCompleted test: replace-before-active
7152025-02-27T17:36:28.982ZAll tests have completed, stopping all downstairs
7162025-02-27T17:36:29.040Z
7172025-02-27T17:36:29.044ZAll Tests have passed
7182025-02-27T17:36:29.044Z13:24 Test duration
7192025-02-27T17:36:29.044Z
7202025-02-27T17:36:29.044Zreal 13:23.738164569
7212025-02-27T17:36:29.044Zuser 18:33.833186773
7222025-02-27T17:36:29.044Zsys 24.465474366
7232025-02-27T17:36:29.044Ztrap 0.520047259
7242025-02-27T17:36:29.044Ztflt 0.038748344
7252025-02-27T17:36:29.045Zdflt 0.029903148
7262025-02-27T17:36:29.045Zkflt 0.001414631
7272025-02-27T17:36:29.045Zlock 25:36:40.875931063
7282025-02-27T17:36:29.045Zslp 1:14:29.947646457
7292025-02-27T17:36:29.045Zlat 2:02.364924762
7302025-02-27T17:36:29.045Zstop 0.215203773
7312025-02-27T17:36:29.045Z+ echo 'test-up-2region-encrypted ends'
7322025-02-27T17:36:29.045Ztest-up-2region-encrypted ends
7332025-02-27T17:36:34.047Zprocess exited: duration 812116 ms, exit code 0
7342025-02-27T17:36:34.047Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
7352025-02-27T17:37:34.089Zexec warning: : stdout descriptor may be held open by a background process; giving up!
7362025-02-27T17:37:34.089Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
7372025-02-27T17:37:34.099Zfound 17 output files
7382025-02-27T17:37:34.099Zuploading: /tmp/test_up-build/dsc-out.txt (48893 bytes)
7392025-02-27T17:37:35.110Zuploaded: /tmp/test_up-build/dsc-out.txt
7402025-02-27T17:37:35.110Zuploading: /tmp/test_up-build/test_up_out.txt (8532110 bytes)
7412025-02-27T17:37:36.201Zuploaded: /tmp/test_up-build/test_up_out.txt
7422025-02-27T17:37:36.201Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (147227 bytes)
7432025-02-27T17:37:37.215Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
7442025-02-27T17:37:37.216Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (144627 bytes)
7452025-02-27T17:37:38.228Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
7462025-02-27T17:37:38.228Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (251009 bytes)
7472025-02-27T17:37:39.243Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
7482025-02-27T17:37:39.245Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (139405 bytes)
7492025-02-27T17:37:40.254Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
7502025-02-27T17:37:40.257Zuploading: /tmp/test_up-build/dsc/downstairs-8850.txt (2481 bytes)
7512025-02-27T17:37:41.263Zuploaded: /tmp/test_up-build/dsc/downstairs-8850.txt
7522025-02-27T17:37:41.266Zuploading: /tmp/test_up-build/dsc/downstairs-8860.txt (33772 bytes)
7532025-02-27T17:37:42.273Zuploaded: /tmp/test_up-build/dsc/downstairs-8860.txt
7542025-02-27T17:37:42.275Zuploading: /tmp/test_up-build/dsc/downstairs-8870.txt (735742 bytes)
7552025-02-27T17:37:43.295Zuploaded: /tmp/test_up-build/dsc/downstairs-8870.txt
7562025-02-27T17:37:43.295Zuploading: /tmp/debug/df.txt (1270 bytes)
7572025-02-27T17:37:44.304Zuploaded: /tmp/debug/df.txt
7582025-02-27T17:37:44.304Zuploading: /tmp/debug/dtrace.txt (966101 bytes)
7592025-02-27T17:37:45.327Zuploaded: /tmp/debug/dtrace.txt
7602025-02-27T17:37:45.330Zuploading: /tmp/debug/iostat.txt (261631 bytes)
7612025-02-27T17:37:45.339Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 261631 -> 265243
7622025-02-27T17:37:46.469Zuploaded: /tmp/debug/iostat.txt
7632025-02-27T17:37:46.469Zuploading: /tmp/debug/mpstat.txt (634385 bytes)
7642025-02-27T17:37:46.484Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 634385 -> 643875
7652025-02-27T17:37:47.491Zuploaded: /tmp/debug/mpstat.txt
7662025-02-27T17:37:47.494Zuploading: /tmp/debug/paging.txt (114550 bytes)
7672025-02-27T17:37:47.501Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 114550 -> 116490
7682025-02-27T17:37:48.507Zuploaded: /tmp/debug/paging.txt
7692025-02-27T17:37:48.510Zuploading: /tmp/debug/prstat.txt (1234063 bytes)
7702025-02-27T17:37:48.530Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 1234063 -> 1252142
7712025-02-27T17:37:49.536Zuploaded: /tmp/debug/prstat.txt
7722025-02-27T17:37:49.539Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
7732025-02-27T17:37:50.545Zuploaded: /tmp/debug/psrinfo.txt
7742025-02-27T17:37:50.549Zuploading: /tmp/debug/upstairs-info.txt (180085 bytes)
7752025-02-27T17:37:51.558Zuploaded: /tmp/debug/upstairs-info.txt