01KJV62YW57YY8CPEA4Q5R5852: test-up-2region-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01KJV63D745FDXKG3WW14Q08BF

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12026-03-04T01:27:45.050Zjob dependencies complete; ready to run (waiting for 21 m 22 s)
22026-03-04T01:28:44.400Zjob assigned to worker 01KJV7AWFCETBFSZKPD347CNKS [factory aws, i-0b1c4ab4614a12d09] (queued for 59 s)
32026-03-04T01:28:52.331Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42026-03-04T01:28:54.540Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52026-03-04T01:28:54.540Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62026-03-04T01:28:56.053Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72026-03-04T01:28:56.053Zdownloading input: /input/build/work/bins/crutest.gz
82026-03-04T01:28:57.548Zdownloaded input: /input/build/work/bins/crutest.gz
92026-03-04T01:28:57.548Zdownloading input: /input/build/work/bins/dsc.gz
102026-03-04T01:28:58.290Zdownloaded input: /input/build/work/bins/dsc.gz
112026-03-04T01:28:58.294Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122026-03-04T01:28:58.318Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132026-03-04T01:28:58.318Zdownloading input: /input/build/work/scripts/test_ds.sh
142026-03-04T01:28:58.347Zdownloaded input: /input/build/work/scripts/test_ds.sh
152026-03-04T01:28:58.347Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162026-03-04T01:28:58.371Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172026-03-04T01:28:58.371Zdownloading input: /input/build/work/scripts/test_repair.sh
182026-03-04T01:28:58.412Zdownloaded input: /input/build/work/scripts/test_repair.sh
192026-03-04T01:28:58.413Zdownloading input: /input/build/work/scripts/test_replay.sh
202026-03-04T01:28:58.443Zdownloaded input: /input/build/work/scripts/test_replay.sh
212026-03-04T01:28:58.444Zdownloading input: /input/build/work/scripts/test_up.sh
222026-03-04T01:28:58.473Zdownloaded input: /input/build/work/scripts/test_up.sh
232026-03-04T01:28:58.473Zdownloading input: /input/build/work/scripts/upstairs_info.d
242026-03-04T01:28:58.504Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252026-03-04T01:28:58.509Zdownloading input: /input/build/tmp/cargo-test-out.log
262026-03-04T01:28:59.051Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272026-03-04T01:28:59.051Zstarting task 0: "setup"
282026-03-04T01:28:59.061Z++ uname -s
292026-03-04T01:28:59.066Z+ kern=SunOS
302026-03-04T01:28:59.066Z+ build_user=build
312026-03-04T01:28:59.066Z+ build_uid=12345
322026-03-04T01:28:59.066Z+ work_dir=/work
332026-03-04T01:28:59.066Z+ input_dir=/input
342026-03-04T01:28:59.067Z+ [[ 0 == 12345 ]]
352026-03-04T01:28:59.067Z+ case "$kern" in
362026-03-04T01:28:59.067Z+ groupadd -g 12345 build
372026-03-04T01:28:59.067Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382026-03-04T01:29:01.072Z+ zfs create -o mountpoint=/work rpool/work
392026-03-04T01:29:01.517Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402026-03-04T01:29:01.523Z+ home_fs=zfs
412026-03-04T01:29:01.523Z+ [[ zfs == autofs ]]
422026-03-04T01:29:01.523Z+ mkdir -p /home/build
432026-03-04T01:29:01.523Z+ chown build:build /home/build /work
442026-03-04T01:29:02.524Z+ chmod 0700 /home/build /work
452026-03-04T01:29:02.529Zprocess exited: duration 3476 ms, exit code 0
 
462026-03-04T01:29:02.538Zstarting task 1: "authentication"
472026-03-04T01:29:02.561Zprocess exited: duration 21 ms, exit code 0
 
482026-03-04T01:29:02.568Zstarting task 2: "build"
492026-03-04T01:29:02.574Z+ banner cores
502026-03-04T01:29:02.577Z
512026-03-04T01:29:02.577Z #### #### ##### ###### ####
522026-03-04T01:29:02.577Z # # # # # # # #
532026-03-04T01:29:02.577Z # # # # # ##### ####
542026-03-04T01:29:02.578Z # # # ##### # #
552026-03-04T01:29:02.578Z # # # # # # # # #
562026-03-04T01:29:02.578Z #### #### # # ###### ####
572026-03-04T01:29:02.578Z
582026-03-04T01:29:02.578Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
592026-03-04T01:29:02.587Z+ banner unpack
602026-03-04T01:29:02.590Z
612026-03-04T01:29:02.590Z # # # # ##### ## #### # #
622026-03-04T01:29:02.590Z # # ## # # # # # # # # #
632026-03-04T01:29:02.590Z # # # # # # # # # # ####
642026-03-04T01:29:02.590Z # # # # # ##### ###### # # #
652026-03-04T01:29:02.590Z # # # ## # # # # # # #
662026-03-04T01:29:02.591Z #### # # # # # #### # #
672026-03-04T01:29:02.591Z
682026-03-04T01:29:02.591Z+ mkdir -p /var/tmp/bins
692026-03-04T01:29:02.595Z+ for t in "$input/bins/"*.gz
702026-03-04T01:29:02.595Z++ basename /input/build/work/bins/crucible-downstairs.gz
712026-03-04T01:29:02.595Z+ b=crucible-downstairs.gz
722026-03-04T01:29:02.595Z+ b=crucible-downstairs
732026-03-04T01:29:02.595Z+ gunzip
742026-03-04T01:29:05.768Z+ chmod +x /var/tmp/bins/crucible-downstairs
752026-03-04T01:29:05.771Z+ for t in "$input/bins/"*.gz
762026-03-04T01:29:05.774Z++ basename /input/build/work/bins/crucible-hammer.gz
772026-03-04T01:29:05.774Z+ b=crucible-hammer.gz
782026-03-04T01:29:05.774Z+ b=crucible-hammer
792026-03-04T01:29:05.774Z+ gunzip
802026-03-04T01:29:08.686Z+ chmod +x /var/tmp/bins/crucible-hammer
812026-03-04T01:29:08.690Z+ for t in "$input/bins/"*.gz
822026-03-04T01:29:08.690Z++ basename /input/build/work/bins/crutest.gz
832026-03-04T01:29:08.695Z+ b=crutest.gz
842026-03-04T01:29:08.695Z+ b=crutest
852026-03-04T01:29:08.695Z+ gunzip
862026-03-04T01:29:11.733Z+ chmod +x /var/tmp/bins/crutest
872026-03-04T01:29:11.736Z+ for t in "$input/bins/"*.gz
882026-03-04T01:29:11.736Z++ basename /input/build/work/bins/dsc.gz
892026-03-04T01:29:11.740Z+ b=dsc.gz
902026-03-04T01:29:11.740Z+ b=dsc
912026-03-04T01:29:11.740Z+ gunzip
922026-03-04T01:29:13.190Z+ chmod +x /var/tmp/bins/dsc
932026-03-04T01:29:13.193Z+ export BINDIR=/var/tmp/bins
942026-03-04T01:29:13.193Z+ BINDIR=/var/tmp/bins
952026-03-04T01:29:13.193Z+ jobpid=1157
962026-03-04T01:29:13.193Z+ echo 'Setup debug logging'
972026-03-04T01:29:13.193ZSetup debug logging
982026-03-04T01:29:13.193Z+ mkdir /tmp/debug
992026-03-04T01:29:13.196Z+ sleep 3600
1002026-03-04T01:29:13.196Z+ psrinfo -v
1012026-03-04T01:29:13.199Z+ df -h
1022026-03-04T01:29:13.207Z+ prstat -d d -mLc 1
1032026-03-04T01:29:13.210Z+ iostat -T d -xn 1
1042026-03-04T01:29:13.210Z+ mpstat -T d 1
1052026-03-04T01:29:13.210Z+ vmstat -T d -p 1
1062026-03-04T01:29:13.212Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1072026-03-04T01:29:13.212Z+ banner test_up_2r_encrypted
1082026-03-04T01:29:13.212Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1092026-03-04T01:29:13.216Z #####
1102026-03-04T01:29:13.217Z ##### ###### #### ##### # # ##### # # #####
1112026-03-04T01:29:13.217Z # # # # # # # # # # #
1122026-03-04T01:29:13.217Z # ##### #### # # # # # ##### # #
1132026-03-04T01:29:13.217Z # # # # # # ##### # #####
1142026-03-04T01:29:13.217Z # # # # # # # # # # #
1152026-03-04T01:29:13.217Z # ###### #### # ####### #### # ####### ####### # #
1162026-03-04T01:29:13.217Z
1172026-03-04T01:29:13.217Z+ ptime -m bash /input/build/work/scripts/test_up.sh -r 2 -N encrypted
1182026-03-04T01:29:13.268Z/input/build/work
1192026-03-04T01:29:13.268ZUsing 2 region sets
1202026-03-04T01:29:13.268ZTurn off color for downstairs dump
1212026-03-04T01:29:13.268ZUpstairs using key: U3m+hJuDyHtzhIsAgKMMxT0AcwU2hqkZLdvWRyo6p0c=
1222026-03-04T01:29:13.268Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1232026-03-04T01:29:13.269ZCreating 6 downstairs regions
1242026-03-04T01:29:13.635ZStarting 6 downstairs
1252026-03-04T01:29:13.638Zdsc started at PID: 1205
1262026-03-04T01:29:18.802ZDisable automatic restart on all downstairs
1272026-03-04T01:29:18.819Z
1282026-03-04T01:29:18.820ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1292026-03-04T01:29:18.830ZRunning test: span
1302026-03-04T01:29:22.966ZCompleted test: span
1312026-03-04T01:29:27.972ZRunning test: big
1322026-03-04T01:29:32.534ZCompleted test: big
1332026-03-04T01:29:37.541ZRunning test: dep
1342026-03-04T01:30:11.251ZCompleted test: dep
1352026-03-04T01:30:16.258ZRunning test: balloon
1362026-03-04T01:34:34.164ZCompleted test: balloon
1372026-03-04T01:34:39.171ZRunning test: deactivate
1382026-03-04T01:34:43.873ZCompleted test: deactivate
1392026-03-04T01:34:48.880ZRunning hammer
1402026-03-04T01:35:07.062ZRun repair tests
1412026-03-04T01:35:07.066Z/var/tmp/bins/crutest fill -g 31 -q --key U3m+hJuDyHtzhIsAgKMMxT0AcwU2hqkZLdvWRyo6p0c= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1422026-03-04T01:35:07.094Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.629830253Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"task":"crutest"}
1432026-03-04T01:35:07.097Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.630445166Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"task":"crutest"}
1442026-03-04T01:35:07.097Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.632275464Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"task":"crutest"}
1452026-03-04T01:35:07.097Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.632545486Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248}
1462026-03-04T01:35:07.097Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"82af9731ff897765f757d37a6e9e2daa9b88f13e\",\n git_commit_timestamp: \"2026-03-04T01:05:37.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.632606624Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248}
1472026-03-04T01:35:07.097Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.632636357Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248}
1482026-03-04T01:35:07.097Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.63266614Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248}
1492026-03-04T01:35:07.100Z{"msg":"Crucible 924c05b3-651b-440d-a9ad-59f4d4e206fe has session id: 9740724b-646a-4178-9d17-fb0efc523a97","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.633176261Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1502026-03-04T01:35:07.100Z{"msg":"Upstairs opts: Upstairs UUID: 924c05b3-651b-440d-a9ad-59f4d4e206fe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.63324972Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1512026-03-04T01:35:07.100Z{"msg":"Crucible stats registered with UUID: 924c05b3-651b-440d-a9ad-59f4d4e206fe","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.633287985Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1522026-03-04T01:35:07.100Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:35:06.633306427Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1532026-03-04T01:35:07.100Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.633484098Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1542026-03-04T01:35:07.100Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.633557497Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1552026-03-04T01:35:07.100Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.633605252Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1562026-03-04T01:35:07.100Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.635453072Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"task":"crutest"}
1572026-03-04T01:35:07.100Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.635537513Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248}
1582026-03-04T01:35:07.100Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"82af9731ff897765f757d37a6e9e2daa9b88f13e\",\n git_commit_timestamp: \"2026-03-04T01:05:37.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.635586518Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248}
1592026-03-04T01:35:07.100Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.635608561Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248}
1602026-03-04T01:35:07.100Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.635630824Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248}
1612026-03-04T01:35:07.100Z{"msg":"Crucible 924c05b3-651b-440d-a9ad-59f4d4e206fe has session id: effe3830-cbe3-4cfc-9f72-77376d6ae3a1","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.635989526Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
1622026-03-04T01:35:07.101Z{"msg":"Upstairs opts: Upstairs UUID: 924c05b3-651b-440d-a9ad-59f4d4e206fe, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.636043023Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
1632026-03-04T01:35:07.101Z{"msg":"Crucible stats registered with UUID: 924c05b3-651b-440d-a9ad-59f4d4e206fe","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.636082178Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
1642026-03-04T01:35:07.101Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:35:06.636110601Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
1652026-03-04T01:35:07.101Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.636135404Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
1662026-03-04T01:35:07.101Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.636173468Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
1672026-03-04T01:35:07.101Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.636196731Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
1682026-03-04T01:35:07.101Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.636318426Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"task":"crutest"}
1692026-03-04T01:35:07.105Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.63635719Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248}
1702026-03-04T01:35:07.105Z{"msg":"924c05b3-651b-440d-a9ad-59f4d4e206fe active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.636456832Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1712026-03-04T01:35:07.105Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.636495027Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"client":"0","":"downstairs","session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1722026-03-04T01:35:07.105Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.63652502Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"client":"1","":"downstairs","session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1732026-03-04T01:35:07.105Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.636560795Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"client":"2","":"downstairs","session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1742026-03-04T01:35:07.105Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.636589958Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1752026-03-04T01:35:07.105Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.636619691Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1762026-03-04T01:35:07.105Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.636666857Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1772026-03-04T01:35:07.105Z{"msg":"ds_connection connected from Ok(127.0.0.1:56832)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.636712853Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1782026-03-04T01:35:07.105Z{"msg":"ds_connection connected from Ok(127.0.0.1:46582)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.636787131Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1792026-03-04T01:35:07.105Z{"msg":"ds_connection connected from Ok(127.0.0.1:46121)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.636821935Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1802026-03-04T01:35:07.105Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.638079545Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"client":"2","":"downstairs","session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1812026-03-04T01:35:07.105Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.638124741Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"client":"1","":"downstairs","session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1822026-03-04T01:35:07.105Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.638146943Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"client":"0","":"downstairs","session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1832026-03-04T01:35:07.105Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.63854316Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1842026-03-04T01:35:07.105Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.638580965Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1852026-03-04T01:35:07.105Z{"msg":"[0]R flush_numbers: [3997, 3997, 3997, 3997, 3997]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.638598877Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"downstairs","":"downstairs","session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1862026-03-04T01:35:07.105Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.6386241Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"downstairs","":"downstairs","session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1872026-03-04T01:35:07.105Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.638643172Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"downstairs","":"downstairs","session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1882026-03-04T01:35:07.105Z{"msg":"[1]R flush_numbers: [3997, 3997, 3997, 3997, 3997]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.638667155Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"downstairs","":"downstairs","session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1892026-03-04T01:35:07.105Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.638684877Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"downstairs","":"downstairs","session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1902026-03-04T01:35:07.105Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.63870898Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"downstairs","":"downstairs","session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1912026-03-04T01:35:07.105Z{"msg":"[2]R flush_numbers: [3997, 3997, 3997, 3997, 3997]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.638727413Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"downstairs","":"downstairs","session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1922026-03-04T01:35:07.105Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.638751005Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"downstairs","":"downstairs","session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1932026-03-04T01:35:07.105Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.638769158Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"downstairs","":"downstairs","session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1942026-03-04T01:35:07.105Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.63879347Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"downstairs","":"downstairs","session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1952026-03-04T01:35:07.106Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.638811163Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"downstairs","":"downstairs","session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1962026-03-04T01:35:07.106Z{"msg":"Next flush: 3998","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.638827945Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"downstairs","":"downstairs","session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1972026-03-04T01:35:07.106Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.638856968Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"downstairs","":"downstairs","session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1982026-03-04T01:35:07.106Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.63887662Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
1992026-03-04T01:35:07.106Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.638901353Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"client":"0","":"downstairs","session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
2002026-03-04T01:35:07.106Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.638919015Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"client":"1","":"downstairs","session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
2012026-03-04T01:35:07.106Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.638935587Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"client":"2","":"downstairs","session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
2022026-03-04T01:35:07.106Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.638966281Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
2032026-03-04T01:35:07.106Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.638989924Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
2042026-03-04T01:35:07.106Z{"msg":"924c05b3-651b-440d-a9ad-59f4d4e206fe is now active with session: 9740724b-646a-4178-9d17-fb0efc523a97","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.639052001Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
2052026-03-04T01:35:07.106Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.639096317Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"session_id":"9740724b-646a-4178-9d17-fb0efc523a97"}
2062026-03-04T01:35:07.106Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.639250075Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248}
2072026-03-04T01:35:07.106Z{"msg":"Activated sub_volume 924c05b3-651b-440d-a9ad-59f4d4e206fe","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.639299401Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248}
2082026-03-04T01:35:07.106Z{"msg":"924c05b3-651b-440d-a9ad-59f4d4e206fe active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.639327384Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2092026-03-04T01:35:07.106Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.639394992Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"client":"0","":"downstairs","session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2102026-03-04T01:35:07.106Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.639454439Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248}
2112026-03-04T01:35:07.106Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.639488323Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"client":"1","":"downstairs","session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2122026-03-04T01:35:07.106Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.639531488Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"client":"2","":"downstairs","session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2132026-03-04T01:35:07.106Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.639589295Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2142026-03-04T01:35:07.106Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.63963256Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2152026-03-04T01:35:07.106Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.639667774Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2162026-03-04T01:35:07.106Z{"msg":"ds_connection connected from Ok(127.0.0.1:54185)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.63970911Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2172026-03-04T01:35:07.106Z{"msg":"ds_connection connected from Ok(127.0.0.1:44363)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.639764496Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2182026-03-04T01:35:07.106Z{"msg":"ds_connection connected from Ok(127.0.0.1:36038)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.639818683Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2192026-03-04T01:35:07.110Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.64013159Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"client":"2","":"downstairs","session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2202026-03-04T01:35:07.110Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.640184486Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"client":"1","":"downstairs","session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2212026-03-04T01:35:07.110Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.640327673Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"client":"0","":"downstairs","session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2222026-03-04T01:35:07.110Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.64054829Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2232026-03-04T01:35:07.110Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.640577373Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2242026-03-04T01:35:07.110Z{"msg":"[0]R flush_numbers: [5317, 5316, 5318, 5319, 5318]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.640683775Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"downstairs","":"downstairs","session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2252026-03-04T01:35:07.110Z{"msg":"[0]R generation: [12, 12, 13, 14, 13]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.640714189Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"downstairs","":"downstairs","session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2262026-03-04T01:35:07.110Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.640733872Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"downstairs","":"downstairs","session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2272026-03-04T01:35:07.110Z{"msg":"[1]R flush_numbers: [5317, 5316, 5318, 5319, 5318]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.640763945Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"downstairs","":"downstairs","session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2282026-03-04T01:35:07.110Z{"msg":"[1]R generation: [12, 12, 13, 14, 13]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.640781507Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"downstairs","":"downstairs","session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2292026-03-04T01:35:07.110Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.640856306Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"downstairs","":"downstairs","session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2302026-03-04T01:35:07.110Z{"msg":"[2]R flush_numbers: [5317, 5316, 5318, 5319, 5318]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.640902152Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"downstairs","":"downstairs","session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2312026-03-04T01:35:07.110Z{"msg":"[2]R generation: [12, 12, 13, 14, 13]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.640921964Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"downstairs","":"downstairs","session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2322026-03-04T01:35:07.110Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.640948117Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"downstairs","":"downstairs","session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2332026-03-04T01:35:07.110Z{"msg":"Max found gen is 15","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.640965459Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"downstairs","":"downstairs","session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2342026-03-04T01:35:07.110Z{"msg":"Generation requested: 31 >= found:15","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.640989932Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"downstairs","":"downstairs","session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2352026-03-04T01:35:07.110Z{"msg":"Next flush: 5320","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.641006984Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"downstairs","":"downstairs","session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2362026-03-04T01:35:07.110Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.641030397Z","hostname":"Wait for a query_work_queue command to finish before sending IO
2372026-03-04T01:35:07.110Zw-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"":"downstairs","":"downstairs","session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2382026-03-04T01:35:07.110Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.641052199Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2392026-03-04T01:35:07.110Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.641074402Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"client":"0","":"downstairs","session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2402026-03-04T01:35:07.110Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.641092784Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"client":"1","":"downstairs","session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2412026-03-04T01:35:07.110Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.641116127Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"client":"2","":"downstairs","session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2422026-03-04T01:35:07.110Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.641133879Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2432026-03-04T01:35:07.110Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.641151031Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2442026-03-04T01:35:07.110Z{"msg":"924c05b3-651b-440d-a9ad-59f4d4e206fe is now active with session: effe3830-cbe3-4cfc-9f72-77376d6ae3a1","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.641176034Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2452026-03-04T01:35:07.110Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.641191546Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248,"session_id":"effe3830-cbe3-4cfc-9f72-77376d6ae3a1"}
2462026-03-04T01:35:07.110Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.641214169Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248}
2472026-03-04T01:35:07.110Z{"msg":"Activated sub_volume 924c05b3-651b-440d-a9ad-59f4d4e206fe","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:06.64122854Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248}
2482026-03-04T01:35:07.110ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
2492026-03-04T01:35:07.111ZFill test
2502026-03-04T01:35:07.149ZRead and Verify all blocks (0..100 range:false)
2512026-03-04T01:35:07.230ZWrote out file "/var/tmp/test_up-build/verify_file"
2522026-03-04T01:35:07.230ZCLIENT: Tests done. All submitted work has been ACK'd
2532026-03-04T01:35:07.230Z----------------------------------------------------------------
2542026-03-04T01:35:07.230Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2552026-03-04T01:35:07.230ZStates: Active Active Active
2562026-03-04T01:35:07.230ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2572026-03-04T01:35:07.234Z 1002 Acked Read 50 Done Done Done false
2582026-03-04T01:35:07.234Z STATES DS:0 DS:1 DS:2 TOTAL
2592026-03-04T01:35:07.234Z Sent 0 0 0 0
2602026-03-04T01:35:07.234Z Done 1 1 1 3
2612026-03-04T01:35:07.234Z Skipped 0 0 0 0
2622026-03-04T01:35:07.234Z Error 0 0 0 0
2632026-03-04T01:35:07.234ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2642026-03-04T01:35:07.234ZDownstairs last five completed: 1001 1000
2652026-03-04T01:35:07.234ZUpstairs last five completed: 1002 1001 1000
2662026-03-04T01:35:07.234Z----------------------------------------------------------------
2672026-03-04T01:35:07.234Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2682026-03-04T01:35:07.234ZStates: Active Active Active
2692026-03-04T01:35:07.234ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2702026-03-04T01:35:07.234Z 1002 Acked Read 50 Sent Sent Done false
2712026-03-04T01:35:07.234Z STATES DS:0 DS:1 DS:2 TOTAL
2722026-03-04T01:35:07.234Z Sent 1 1 0 2
2732026-03-04T01:35:07.234Z Done 0 0 1 1
2742026-03-04T01:35:07.234Z Skipped 0 0 0 0
2752026-03-04T01:35:07.234Z Error 0 0 0 0
2762026-03-04T01:35:07.234ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2772026-03-04T01:35:07.234ZDownstairs last five completed: 1001 1000
2782026-03-04T01:35:07.234ZUpstairs last five completed: 1002 1001 1000
2792026-03-04T01:35:07.234ZCLIENT: Up:0 ds:2 act:6
2802026-03-04T01:35:11.232Z----------------------------------------------------------------
2812026-03-04T01:35:11.232Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2822026-03-04T01:35:11.232ZDownstairs last five completed: 1003 1002 1001 1000
2832026-03-04T01:35:11.232ZUpstairs last five completed: 1003 1002 1001 1000
2842026-03-04T01:35:11.235Z----------------------------------------------------------------
2852026-03-04T01:35:11.235Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2862026-03-04T01:35:11.235ZDownstairs last five completed: 1003 1002 1001 1000
2872026-03-04T01:35:11.235ZUpstairs last five completed: 1003 1002 1001 1000
2882026-03-04T01:35:11.235ZCLIENT: Up:0 ds:0 act:6
2892026-03-04T01:35:11.235ZCLIENT: All crucible jobs finished, exiting program
2902026-03-04T01:35:11.235Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:35:10.768250635Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1248}
2912026-03-04T01:35:11.242ZRepair setup passed
2922026-03-04T01:35:11.245ZCopy the region for /var/tmp/test_up-build/8830
2932026-03-04T01:35:11.248Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2942026-03-04T01:35:11.251Z/var/tmp/bins/crutest fill -g 32 -q --key U3m+hJuDyHtzhIsAgKMMxT0AcwU2hqkZLdvWRyo6p0c= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2952026-03-04T01:35:11.276Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.812167336Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"task":"crutest"}
2962026-03-04T01:35:11.279Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.812807952Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"task":"crutest"}
2972026-03-04T01:35:11.279Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.81463746Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"task":"crutest"}
2982026-03-04T01:35:11.279Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.814904402Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254}
2992026-03-04T01:35:11.279Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"82af9731ff897765f757d37a6e9e2daa9b88f13e\",\n git_commit_timestamp: \"2026-03-04T01:05:37.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.814989282Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254}
3002026-03-04T01:35:11.279Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.815043238Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254}
3012026-03-04T01:35:11.279Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.815081223Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254}
3022026-03-04T01:35:11.282Z{"msg":"Crucible 18292481-0d2d-47f4-a06e-8416163d5fdb has session id: e7992df9-8593-45f2-aa27-82d9e83e3a3f","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.815543278Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3032026-03-04T01:35:11.282Z{"msg":"Upstairs opts: Upstairs UUID: 18292481-0d2d-47f4-a06e-8416163d5fdb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.815607565Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3042026-03-04T01:35:11.282Z{"msg":"Crucible stats registered with UUID: 18292481-0d2d-47f4-a06e-8416163d5fdb","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.81564669Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3052026-03-04T01:35:11.282Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:35:10.815665522Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3062026-03-04T01:35:11.282Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.815800688Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3072026-03-04T01:35:11.282Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.815858325Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3082026-03-04T01:35:11.282Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.815881738Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3092026-03-04T01:35:11.282Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.817737399Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"task":"crutest"}
3102026-03-04T01:35:11.282Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.817794336Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254}
3112026-03-04T01:35:11.282Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"82af9731ff897765f757d37a6e9e2daa9b88f13e\",\n git_commit_timestamp: \"2026-03-04T01:05:37.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.817850693Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254}
3122026-03-04T01:35:11.283Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.817881716Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254}
3132026-03-04T01:35:11.283Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.817932282Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254}
3142026-03-04T01:35:11.283Z{"msg":"Crucible 18292481-0d2d-47f4-a06e-8416163d5fdb has session id: 91035a25-db8a-4d1e-b23d-f4014b2b1537","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.818312497Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3152026-03-04T01:35:11.283Z{"msg":"Upstairs opts: Upstairs UUID: 18292481-0d2d-47f4-a06e-8416163d5fdb, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.818371875Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3162026-03-04T01:35:11.283Z{"msg":"Crucible stats registered with UUID: 18292481-0d2d-47f4-a06e-8416163d5fdb","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.818405489Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3172026-03-04T01:35:11.283Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:35:10.818439403Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3182026-03-04T01:35:11.283Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.818474857Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3192026-03-04T01:35:11.287Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.818524553Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3202026-03-04T01:35:11.287Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.818563407Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3212026-03-04T01:35:11.287Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.818590581Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"task":"crutest"}
3222026-03-04T01:35:11.287Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.818627765Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254}
3232026-03-04T01:35:11.287Z{"msg":"18292481-0d2d-47f4-a06e-8416163d5fdb active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.818771772Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3242026-03-04T01:35:11.287Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.818805826Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"client":"0","":"downstairs","session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3252026-03-04T01:35:11.287Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.818829049Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"client":"1","":"downstairs","session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3262026-03-04T01:35:11.287Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.818847281Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"client":"2","":"downstairs","session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3272026-03-04T01:35:11.287Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.818864903Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3282026-03-04T01:35:11.287Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.818882345Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3292026-03-04T01:35:11.287Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.818899438Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3302026-03-04T01:35:11.287Z{"msg":"ds_connection connected from Ok(127.0.0.1:60803)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.819078619Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3312026-03-04T01:35:11.287Z{"msg":"ds_connection connected from Ok(127.0.0.1:63459)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.819123724Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3322026-03-04T01:35:11.287Z{"msg":"ds_connection connected from Ok(127.0.0.1:43019)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.819232567Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3332026-03-04T01:35:11.287Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.820308995Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"client":"0","":"downstairs","session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3342026-03-04T01:35:11.287Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.820353161Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"client":"2","":"downstairs","session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3352026-03-04T01:35:11.287Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.820375893Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"client":"1","":"downstairs","session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3362026-03-04T01:35:11.287Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.82076967Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3372026-03-04T01:35:11.287Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.820809065Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3382026-03-04T01:35:11.287Z{"msg":"[0]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.820831908Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"downstairs","":"downstairs","session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3392026-03-04T01:35:11.287Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.820866382Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"downstairs","":"downstairs","session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3402026-03-04T01:35:11.287Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.820901786Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"downstairs","":"downstairs","session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3412026-03-04T01:35:11.288Z{"msg":"[1]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.82093835Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"downstairs","":"downstairs","session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3422026-03-04T01:35:11.288Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.820973525Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"downstairs","":"downstairs","session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3432026-03-04T01:35:11.288Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.821011109Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"downstairs","":"downstairs","session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3442026-03-04T01:35:11.288Z{"msg":"[2]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.821044463Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"downstairs","":"downstairs","session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3452026-03-04T01:35:11.288Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.821081507Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"downstairs","":"downstairs","session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3462026-03-04T01:35:11.288Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.821117582Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"downstairs","":"downstairs","session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3472026-03-04T01:35:11.288Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.821163857Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"downstairs","":"downstairs","session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3482026-03-04T01:35:11.288Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.821212623Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"downstairs","":"downstairs","session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3492026-03-04T01:35:11.288Z{"msg":"Next flush: 3999","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.82127511Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"downstairs","":"downstairs","session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3502026-03-04T01:35:11.288Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.821314995Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"downstairs","":"downstairs","session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3512026-03-04T01:35:11.288Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.82135247Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3522026-03-04T01:35:11.288Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.82152032Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"client":"0","":"downstairs","session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3532026-03-04T01:35:11.288Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.821575606Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"client":"1","":"downstairs","session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3542026-03-04T01:35:11.288Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.8216096Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"client":"2","":"downstairs","session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3552026-03-04T01:35:11.288Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.821661267Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3562026-03-04T01:35:11.288Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.821771709Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3572026-03-04T01:35:11.288Z{"msg":"18292481-0d2d-47f4-a06e-8416163d5fdb is now active with session: e7992df9-8593-45f2-aa27-82d9e83e3a3f","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.821821315Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3582026-03-04T01:35:11.288Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.821902745Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"session_id":"e7992df9-8593-45f2-aa27-82d9e83e3a3f"}
3592026-03-04T01:35:11.288Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.82194164Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254}
3602026-03-04T01:35:11.288Z{"msg":"Activated sub_volume 18292481-0d2d-47f4-a06e-8416163d5fdb","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.821959452Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254}
3612026-03-04T01:35:11.288Z{"msg":"18292481-0d2d-47f4-a06e-8416163d5fdb active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.821981735Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3622026-03-04T01:35:11.288Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.822005117Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254}
3632026-03-04T01:35:11.288Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.822037901Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"client":"0","":"downstairs","session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3642026-03-04T01:35:11.288Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.822059964Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"client":"1","":"downstairs","session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3652026-03-04T01:35:11.288Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.822076906Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"client":"2","":"downstairs","session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3662026-03-04T01:35:11.288Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.822100949Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3672026-03-04T01:35:11.288Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.822118371Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3682026-03-04T01:35:11.288Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.822142414Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3692026-03-04T01:35:11.288Z{"msg":"ds_connection connected from Ok(127.0.0.1:51452)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.822189179Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3702026-03-04T01:35:11.288Z{"msg":"ds_connection connected from Ok(127.0.0.1:36515)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.822239945Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3712026-03-04T01:35:11.288Z{"msg":"ds_connection connected from Ok(127.0.0.1:39876)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.822265278Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3722026-03-04T01:35:11.288Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.822306483Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"client":"2","":"downstairs","session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3732026-03-04T01:35:11.288Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.822332596Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"client":"0","":"downstairs","session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3742026-03-04T01:35:11.292Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.822541051Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"client":"1","":"downstairs","session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3752026-03-04T01:35:11.292Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.822719622Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3762026-03-04T01:35:11.292Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.822768708Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3772026-03-04T01:35:11.292Z{"msg":"[0]R flush_numbers: [5320, 5320, 5320, 5320, 5320]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.822916336Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"downstairs","":"downstairs","session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3782026-03-04T01:35:11.292Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.822959531Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"downstairs","":"downstairs","session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3792026-03-04T01:35:11.292Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.823014608Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"downstairs","":"downstairs","session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3802026-03-04T01:35:11.292Z{"msg":"[1]R flush_numbers: [5320, 5320, 5320, 5320, 5320]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.823055182Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"downstairs","":"downstairs","session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3812026-03-04T01:35:11.292Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.823092397Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"downstairs","":"downstairs","session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3822026-03-04T01:35:11.292Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
3832026-03-04T01:35:11.292Z,"time":"2026-03-04T01:35:10.82311842Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"downstairs","":"downstairs","session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3842026-03-04T01:35:11.292Z{"msg":"[2]R flush_numbers: [5320, 5320, 5320, 5320, 5320]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.823157045Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"downstairs","":"downstairs","session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3852026-03-04T01:35:11.292Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.823183658Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"downstairs","":"downstairs","session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3862026-03-04T01:35:11.292Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.823214201Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"downstairs","":"downstairs","session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3872026-03-04T01:35:11.292Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.823237064Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"downstairs","":"downstairs","session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3882026-03-04T01:35:11.292Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.823269488Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"downstairs","":"downstairs","session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3892026-03-04T01:35:11.292Z{"msg":"Next flush: 5321","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.823292931Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"downstairs","":"downstairs","session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3902026-03-04T01:35:11.292Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.823346977Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"":"downstairs","":"downstairs","session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3912026-03-04T01:35:11.292Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.823393723Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3922026-03-04T01:35:11.292Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.823412685Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"client":"0","":"downstairs","session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3932026-03-04T01:35:11.292Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.823450149Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"client":"1","":"downstairs","session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3942026-03-04T01:35:11.292Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.823468562Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"client":"2","":"downstairs","session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3952026-03-04T01:35:11.292Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.823494075Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3962026-03-04T01:35:11.292Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.823511107Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3972026-03-04T01:35:11.292Z{"msg":"18292481-0d2d-47f4-a06e-8416163d5fdb is now active with session: 91035a25-db8a-4d1e-b23d-f4014b2b1537","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.82354126Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3982026-03-04T01:35:11.292Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.823557142Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
3992026-03-04T01:35:11.292Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,Disk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
4002026-03-04T01:35:11.293Z"time":"2026-03-04T01:35:10.823573614Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254}
4012026-03-04T01:35:11.293Z{"msg":"Activated sub_volume 18292481-0d2d-47f4-a06e-8416163d5fdb","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:10.823591716Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254}
4022026-03-04T01:35:11.293ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4032026-03-04T01:35:11.293ZFill test
4042026-03-04T01:35:11.341ZRead and Verify all blocks (0..100 range:false)
4052026-03-04T01:35:11.420ZWrote out file "/var/tmp/test_up-build/verify_file"
4062026-03-04T01:35:11.420ZCLIENT: Tests done. All submitted work has been ACK'd
4072026-03-04T01:35:11.423Z----------------------------------------------------------------
4082026-03-04T01:35:11.423Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4092026-03-04T01:35:11.423ZStates: Active Active Active
4102026-03-04T01:35:11.423ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4112026-03-04T01:35:11.423Z 1002 Acked Read 50 Done Done Done false
4122026-03-04T01:35:11.423Z STATES DS:0 DS:1 DS:2 TOTAL
4132026-03-04T01:35:11.423Z Sent 0 0 0 0
4142026-03-04T01:35:11.423Z Done 1 1 1 3
4152026-03-04T01:35:11.423Z Skipped 0 0 0 0
4162026-03-04T01:35:11.423Z Error 0 0 0 0
4172026-03-04T01:35:11.423ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4182026-03-04T01:35:11.423ZDownstairs last five completed: 1001 1000
4192026-03-04T01:35:11.423ZUpstairs last five completed: 1002 1001 1000
4202026-03-04T01:35:11.423Z----------------------------------------------------------------
4212026-03-04T01:35:11.423Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4222026-03-04T01:35:11.423ZStates: Active Active Active
4232026-03-04T01:35:11.424ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4242026-03-04T01:35:11.424Z 1002 Acked Read 50 Sent Done Sent false
4252026-03-04T01:35:11.424Z STATES DS:0 DS:1 DS:2 TOTAL
4262026-03-04T01:35:11.424Z Sent 1 0 1 2
4272026-03-04T01:35:11.424Z Done 0 1 0 1
4282026-03-04T01:35:11.424Z Skipped 0 0 0 0
4292026-03-04T01:35:11.424Z Error 0 0 0 0
4302026-03-04T01:35:11.424ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4312026-03-04T01:35:11.424ZDownstairs last five completed: 1001 1000
4322026-03-04T01:35:11.424ZUpstairs last five completed: 1002 1001 1000
4332026-03-04T01:35:11.424ZCLIENT: Up:0 ds:2 act:6
4342026-03-04T01:35:15.421Z----------------------------------------------------------------
4352026-03-04T01:35:15.421Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4362026-03-04T01:35:15.421ZDownstairs last five completed: 1003 1002 1001 1000
4372026-03-04T01:35:15.421ZUpstairs last five completed: 1003 1002 1001 1000
4382026-03-04T01:35:15.424Z----------------------------------------------------------------
4392026-03-04T01:35:15.424Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4402026-03-04T01:35:15.424ZDownstairs last five completed: 1003 1002 1001 1000
4412026-03-04T01:35:15.424ZUpstairs last five completed: 1003 1002 1001 1000
4422026-03-04T01:35:15.425ZCLIENT: Up:0 ds:0 act:6
4432026-03-04T01:35:15.425ZCLIENT: All crucible jobs finished, exiting program
4442026-03-04T01:35:15.425Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:35:14.958121934Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254}
4452026-03-04T01:35:15.425Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:35:14.958345461Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
4462026-03-04T01:35:15.425Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:35:14.958401057Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
4472026-03-04T01:35:15.425Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:35:14.958464225Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
4482026-03-04T01:35:15.425Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:35:14.958502479Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
4492026-03-04T01:35:15.425Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:35:14.958539114Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
4502026-03-04T01:35:15.425Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:35:14.958574308Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
4512026-03-04T01:35:15.425Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:35:14.958614963Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
4522026-03-04T01:35:15.425Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:35:14.958652767Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
4532026-03-04T01:35:15.425Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:35:14.958686531Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254,"session_id":"91035a25-db8a-4d1e-b23d-f4014b2b1537"}
4542026-03-04T01:35:15.425Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:35:14.958717015Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1254}
4552026-03-04T01:35:15.428ZRepair part 1 passed
4562026-03-04T01:35:15.428Z
4572026-03-04T01:35:15.428ZKill the current downstairs
4582026-03-04T01:35:15.501ZDownstairs 2 stopped
4592026-03-04T01:35:15.501Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
4602026-03-04T01:35:15.504ZNow put back the original so we have a mismatch
4612026-03-04T01:35:15.504Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
4622026-03-04T01:35:15.510ZRestart downstairs with old directory
4632026-03-04T01:35:15.529Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4642026-03-04T01:35:15.547ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4652026-03-04T01:35:15.550Z 0 000-009 32 32 31 3999 3999 3998 F F F <---
4662026-03-04T01:35:15.550Z 1 010-019 32 32 31 3999 3999 3998 F F F <---
4672026-03-04T01:35:15.550Z 2 020-029 32 32 31 3999 3999 3998 F F F <---
4682026-03-04T01:35:15.551Z 3 030-039 32 32 31 3999 3999 3998 F F F <---
4692026-03-04T01:35:15.551Z 4 040-049 32 32 31 3999 3999 3998 F F F <---
4702026-03-04T01:35:15.551ZMax gen: 32, Max flush: 3999
4712026-03-04T01:35:15.551ZError: Difference in extent metadata found!
4722026-03-04T01:35:15.554Zdump test found error as expected
4732026-03-04T01:35:15.554Z
4742026-03-04T01:35:15.554Z
4752026-03-04T01:35:15.554Z/var/tmp/bins/crutest verify --range -g 33 -q --key U3m+hJuDyHtzhIsAgKMMxT0AcwU2hqkZLdvWRyo6p0c= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
4762026-03-04T01:35:15.580Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.116310177Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"task":"crutest"}
4772026-03-04T01:35:15.583Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.116933381Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"task":"crutest"}
4782026-03-04T01:35:15.583Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.118836658Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"task":"crutest"}
4792026-03-04T01:35:15.583Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.119177049Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262}
4802026-03-04T01:35:15.583Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"82af9731ff897765f757d37a6e9e2daa9b88f13e\",\n git_commit_timestamp: \"2026-03-04T01:05:37.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.119257338Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262}
4812026-03-04T01:35:15.583Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.119308624Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262}
4822026-03-04T01:35:15.583Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.119354579Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262}
4832026-03-04T01:35:15.586Z{"msg":"Crucible 545d0f1f-f828-4b98-9118-3dad8164fc4a has session id: a700365c-76bb-4042-8bb8-1ad05450fe24","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.119840057Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
4842026-03-04T01:35:15.586Z{"msg":"Upstairs opts: Upstairs UUID: 545d0f1f-f828-4b98-9118-3dad8164fc4a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.119920277Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
4852026-03-04T01:35:15.586Z{"msg":"Crucible stats registered with UUID: 545d0f1f-f828-4b98-9118-3dad8164fc4a","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.119967032Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
4862026-03-04T01:35:15.586Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:35:15.119988555Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
4872026-03-04T01:35:15.587Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.120132152Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
4882026-03-04T01:35:15.587Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.120205371Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
4892026-03-04T01:35:15.587Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.120249366Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
4902026-03-04T01:35:15.587Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.12204625Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"task":"crutest"}
4912026-03-04T01:35:15.587Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.122114078Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262}
4922026-03-04T01:35:15.587Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"82af9731ff897765f757d37a6e9e2daa9b88f13e\",\n git_commit_timestamp: \"2026-03-04T01:05:37.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.122174195Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262}
4932026-03-04T01:35:15.587Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.12221295Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262}
4942026-03-04T01:35:15.587Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.122251445Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262}
4952026-03-04T01:35:15.587Z{"msg":"Crucible 545d0f1f-f828-4b98-9118-3dad8164fc4a has session id: 8edf150c-0597-4f16-918b-78c1aff74ffd","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.122539729Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
4962026-03-04T01:35:15.587Z{"msg":"Upstairs opts: Upstairs UUID: 545d0f1f-f828-4b98-9118-3dad8164fc4a, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.122606817Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
4972026-03-04T01:35:15.587Z{"msg":"Crucible stats registered with UUID: 545d0f1f-f828-4b98-9118-3dad8164fc4a","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.122646362Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
4982026-03-04T01:35:15.587Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:35:15.122666944Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
4992026-03-04T01:35:15.587Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.122705109Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
5002026-03-04T01:35:15.587Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.122732702Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
5012026-03-04T01:35:15.587Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.122765476Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
5022026-03-04T01:35:15.590Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.122980411Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"task":"crutest"}
5032026-03-04T01:35:15.590Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.123025327Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262}
5042026-03-04T01:35:15.590Z{"msg":"545d0f1f-f828-4b98-9118-3dad8164fc4a active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.123103666Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5052026-03-04T01:35:15.590Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.123142881Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"client":"0","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5062026-03-04T01:35:15.590Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.123165524Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"client":"1","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5072026-03-04T01:35:15.590Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.123200288Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"client":"2","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5082026-03-04T01:35:15.590Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.12322075Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5092026-03-04T01:35:15.590Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.123264245Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5102026-03-04T01:35:15.590Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.123313201Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5112026-03-04T01:35:15.590Z{"msg":"ds_connection connected from Ok(127.0.0.1:42332)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.123346545Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5122026-03-04T01:35:15.590Z{"msg":"ds_connection connected from Ok(127.0.0.1:61580)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.12339007Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5132026-03-04T01:35:15.590Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:35:15.123439546Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5142026-03-04T01:35:15.590Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:35:15.123586464Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5152026-03-04T01:35:15.591Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:35:15.123648701Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5162026-03-04T01:35:15.591Z{"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.123678795Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5172026-03-04T01:35:15.591Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.123705958Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5182026-03-04T01:35:15.591Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.124699026Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"client":"0","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5192026-03-04T01:35:15.591Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.124767534Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"client":"1","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5202026-03-04T01:35:15.591Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.125077941Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5212026-03-04T01:35:15.591Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:15.125121077Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5222026-03-04T01:35:25.588Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.125026015Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5232026-03-04T01:35:25.592Z{"msg":"ds_connection connected from Ok(127.0.0.1:64000)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.125364735Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5242026-03-04T01:35:25.592Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.126730568Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"client":"2","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5252026-03-04T01:35:25.592Z{"msg":"[0]R flush_numbers: [3999, 3999, 3999, 3999, 3999]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.127215366Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"downstairs","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5262026-03-04T01:35:25.593Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.127255871Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"downstairs","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5272026-03-04T01:35:25.593Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.127275773Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"downstairs","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5282026-03-04T01:35:25.593Z{"msg":"[1]R flush_numbers: [3999, 3999, 3999, 3999, 3999]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.127294135Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"downstairs","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5292026-03-04T01:35:25.593Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.127320228Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"downstairs","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5302026-03-04T01:35:25.593Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.127339061Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"downstairs","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5312026-03-04T01:35:25.593Z{"msg":"[2]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.127357433Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"downstairs","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5322026-03-04T01:35:25.593Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.127381166Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"downstairs","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5332026-03-04T01:35:25.593Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.127398978Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"downstairs","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5342026-03-04T01:35:25.593Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.12741587Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"downstairs","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5352026-03-04T01:35:25.593Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.127439562Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"downstairs","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5362026-03-04T01:35:25.593Z{"msg":"Next flush: 4000","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.127455794Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"downstairs","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5372026-03-04T01:35:25.593Z{"msg":"extent 0 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.127482207Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5382026-03-04T01:35:25.593Z{"msg":"extent:0 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.127517772Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5392026-03-04T01:35:25.593Z{"msg":"found dest for source 0 for extent at index 0 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.127537354Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5402026-03-04T01:35:25.593Z{"msg":"extent 1 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.127563687Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5412026-03-04T01:35:25.593Z{"msg":"extent:1 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.127596521Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5422026-03-04T01:35:25.593Z{"msg":"found dest for source 0 for extent at index 1 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.127615533Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5432026-03-04T01:35:25.593Z{"msg":"extent 2 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.127648727Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5442026-03-04T01:35:25.593Z{"msg":"extent:2 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.127680201Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5452026-03-04T01:35:25.593Z{"msg":"found dest for source 0 for extent at index 2 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.127724116Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5462026-03-04T01:35:25.593Z{"msg":"extent 3 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.127784954Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5472026-03-04T01:35:25.593Z{"msg":"extent:3 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.12784319Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5482026-03-04T01:35:25.593Z{"msg":"found dest for source 0 for extent at index 3 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.127888926Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5492026-03-04T01:35:25.593Z{"msg":"extent 4 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.12791969Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5502026-03-04T01:35:25.593Z{"msg":"extent:4 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.127952974Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5512026-03-04T01:35:25.593Z{"msg":"found dest for source 0 for extent at index 4 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.127971966Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5522026-03-04T01:35:25.593Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.128000139Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"client":"0","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5532026-03-04T01:35:25.593Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.128030893Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"client":"1","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5542026-03-04T01:35:25.593Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.128063847Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"client":"2","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5552026-03-04T01:35:25.593Z{"msg":"Full repair list: {ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.128147887Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"downstairs","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5562026-03-04T01:35:25.597Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 4000, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 4000, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 4000, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 4000, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 4000, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.128402657Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"downstairs","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5572026-03-04T01:35:25.597Z{"msg":"starting reconciliation 5a763ac6-ba9a-4992-8f6e-34bbb033ac3a: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.128452493Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"downstairs","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5582026-03-04T01:35:25.597Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.128483177Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"client":"2","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5592026-03-04T01:35:25.601Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.138279694Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"client":"2","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5602026-03-04T01:35:25.605Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.142155535Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"client":"2","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5612026-03-04T01:35:25.609Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.145927905Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"client":"2","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5622026-03-04T01:35:25.612Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.149592411Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"client":"2","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5632026-03-04T01:35:25.617Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.152723714Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"downstairs","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5642026-03-04T01:35:25.617Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.152766799Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"client":"0","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5652026-03-04T01:35:25.620Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.152787362Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"client":"1","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5662026-03-04T01:35:25.620Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.152822016Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"client":"2","":"downstairs","session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5672026-03-04T01:35:25.620Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.152848829Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5682026-03-04T01:35:25.620Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.152868742Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5692026-03-04T01:35:25.620Z{"msg":"545d0f1f-f828-4b98-9118-3dad8164fc4a is now active with session: a700365c-76bb-4042-8bb8-1ad05450fe24","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.152902875Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"session_id":"a700365c-76bb-4042-8bb8-1ad05450fe24"}
5702026-03-04T01:35:25.620Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.152921488Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262}
5712026-03-04T01:35:25.620Z{"msg":"Activated sub_volume 545d0f1f-f828-4b98-9118-3dad8164fc4a","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.152952321Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262}
5722026-03-04T01:35:25.620Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.152969954Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262}
5732026-03-04T01:35:25.620Z{"msg":"545d0f1f-f828-4b98-9118-3dad8164fc4a active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.152995406Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
5742026-03-04T01:35:25.621Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.153012829Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"client":"0","":"downstairs","session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
5752026-03-04T01:35:25.621Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.153030231Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"client":"1","":"downstairs","session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
5762026-03-04T01:35:25.621Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.153059014Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"client":"2","":"downstairs","session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
5772026-03-04T01:35:25.621Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.153097829Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
5782026-03-04T01:35:25.621Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.153131873Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
5792026-03-04T01:35:25.621Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.153161906Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
5802026-03-04T01:35:25.621Z{"msg":"ds_connection connected from Ok(127.0.0.1:61679)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.15319273Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
5812026-03-04T01:35:25.621Z{"msg":"ds_connection connected from Ok(127.0.0.1:53644)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.153225824Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
5822026-03-04T01:35:25.621Z{"msg":"ds_connection connected from Ok(127.0.0.1:45744)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.153259998Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
5832026-03-04T01:35:25.621Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.154093817Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"client":"2","":"downstairs","session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
5842026-03-04T01:35:25.621Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.154176207Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"client":"1","":"downstairs","session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
5852026-03-04T01:35:25.621Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.154384052Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"client":"0","":"downstairs","session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
5862026-03-04T01:35:25.621Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.154509667Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
5872026-03-04T01:35:25.621Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.154613229Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
5882026-03-04T01:35:25.621Z{"msg":"[0]R flush_numbers: [5321, 5321, 5321, 5321, 5321]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.154722442Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"downstairs","":"downstairs","session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
5892026-03-04T01:35:25.621Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.154753696Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"downstairs","":"downstairs","session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
5902026-03-04T01:35:25.621Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.154773368Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"downstairs","":"downstairs","session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
5912026-03-04T01:35:25.621Z{"msg":"[1]R flush_numbers: [5321, 5321, 5321, 5321, 5321]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.154794681Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"downstairs","":"downstairs","session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
5922026-03-04T01:35:25.621Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.154825885Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"downstairs","":"downstairs","session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
5932026-03-04T01:35:25.621Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.154857228Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"downstairs","":"downstairs","session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
5942026-03-04T01:35:25.621Z{"msg":"[2]R flush_numbers: [5321, 5321, 5321, 5321, 5321]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.154891732Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"Wait for a query_work_queue command to finish before sending IO
5952026-03-04T01:35:25.621Zdownstairs","":"downstairs","session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
5962026-03-04T01:35:25.621Z{"msg":"[2]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.154922546Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"downstairs","":"downstairs","session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
5972026-03-04T01:35:25.621Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.154940338Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"downstairs","":"downstairs","session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
5982026-03-04T01:35:25.621Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.15495661Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"downstairs","":"downstairs","session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
5992026-03-04T01:35:25.621Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.154992595Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"downstairs","":"downstairs","session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
6002026-03-04T01:35:25.621Z{"msg":"Next flush: 5322","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.155031509Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"downstairs","":"downstairs","session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
6012026-03-04T01:35:25.621Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.155061663Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"":"downstairs","":"downstairs","session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
6022026-03-04T01:35:25.621Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.155093027Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
6032026-03-04T01:35:25.621Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.1551201Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"client":"0","":"downstairs","session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
6042026-03-04T01:35:25.621Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.155152323Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"client":"1","":"downstairs","session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
6052026-03-04T01:35:25.621Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.155176746Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"client":"2","":"downstairs","session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
6062026-03-04T01:35:25.621Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.155213471Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":Disk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
6072026-03-04T01:35:25.621Z1262,"session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
6082026-03-04T01:35:25.621Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.155245614Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
6092026-03-04T01:35:25.621Z{"msg":"545d0f1f-f828-4b98-9118-3dad8164fc4a is now active with session: 8edf150c-0597-4f16-918b-78c1aff74ffd","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.155279369Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
6102026-03-04T01:35:25.621Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.155297291Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262,"session_id":"8edf150c-0597-4f16-918b-78c1aff74ffd"}
6112026-03-04T01:35:25.621Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.155321294Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262}
6122026-03-04T01:35:25.622Z{"msg":"Activated sub_volume 545d0f1f-f828-4b98-9118-3dad8164fc4a","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:35:25.155335955Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262}
6132026-03-04T01:35:25.622ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
6142026-03-04T01:35:25.622ZRead and Verify all blocks (0..100 range:true)
6152026-03-04T01:35:25.700ZWrote out file "/var/tmp/test_up-build/verify_file"
6162026-03-04T01:35:25.700ZVerify test completed
6172026-03-04T01:35:25.704ZWrote out file "/var/tmp/test_up-build/verify_file"
6182026-03-04T01:35:25.704ZCLIENT: Tests done. All submitted work has been ACK'd
6192026-03-04T01:35:25.704Z----------------------------------------------------------------
6202026-03-04T01:35:25.704Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6212026-03-04T01:35:25.704ZStates: Active Active Active
6222026-03-04T01:35:25.704ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6232026-03-04T01:35:25.704Z 1000 Acked Read 50 Done Done Done false
6242026-03-04T01:35:25.704Z STATES DS:0 DS:1 DS:2 TOTAL
6252026-03-04T01:35:25.704Z Sent 0 0 0 0
6262026-03-04T01:35:25.704Z Done 1 1 1 3
6272026-03-04T01:35:25.704Z Skipped 0 0 0 0
6282026-03-04T01:35:25.704Z Error 0 0 0 0
6292026-03-04T01:35:25.704ZLast Flush: None None None
6302026-03-04T01:35:25.704ZDownstairs last five completed:
6312026-03-04T01:35:25.704ZUpstairs last five completed: 1000
6322026-03-04T01:35:25.704Z----------------------------------------------------------------
6332026-03-04T01:35:25.704Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6342026-03-04T01:35:25.704ZStates: Active Active Active
6352026-03-04T01:35:25.704ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6362026-03-04T01:35:25.704Z 1000 Acked Read 50 Done Done Done false
6372026-03-04T01:35:25.704Z STATES DS:0 DS:1 DS:2 TOTAL
6382026-03-04T01:35:25.704Z Sent 0 0 0 0
6392026-03-04T01:35:25.704Z Done 1 1 1 3
6402026-03-04T01:35:25.704Z Skipped 0 0 0 0
6412026-03-04T01:35:25.705Z Error 0 0 0 0
6422026-03-04T01:35:25.705ZLast Flush: None None None
6432026-03-04T01:35:25.705ZDownstairs last five completed:
6442026-03-04T01:35:25.705ZUpstairs last five completed: 1000
6452026-03-04T01:35:25.705ZCLIENT: Up:0 ds:2 act:6
6462026-03-04T01:35:29.701Z----------------------------------------------------------------
6472026-03-04T01:35:29.701Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6482026-03-04T01:35:29.701ZDownstairs last five completed: 1001 1000
6492026-03-04T01:35:29.701ZUpstairs last five completed: 1001 1000
6502026-03-04T01:35:29.704Z----------------------------------------------------------------
6512026-03-04T01:35:29.705Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6522026-03-04T01:35:29.705ZDownstairs last five completed: 1001 1000
6532026-03-04T01:35:29.705ZUpstairs last five completed: 1001 1000
6542026-03-04T01:35:29.705ZCLIENT: Up:0 ds:0 act:6
6552026-03-04T01:35:29.705ZCLIENT: All crucible jobs finished, exiting program
6562026-03-04T01:35:29.705Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:35:29.238696219Z","hostname":"w-01KJV7AWFCETBFSZKPD347CNKS","pid":1262}
6572026-03-04T01:35:29.707ZRepair part 2 passed
6582026-03-04T01:35:29.711Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
6592026-03-04T01:35:29.726ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6602026-03-04T01:35:29.726Z 0 000-009 32 32 32 3999 3999 3999 F F F
6612026-03-04T01:35:29.726Z 1 010-019 32 32 32 3999 3999 3999 F F F
6622026-03-04T01:35:29.726Z 2 020-029 32 32 32 3999 3999 3999 F F F
6632026-03-04T01:35:29.726Z 3 030-039 32 32 32 3999 3999 3999 F F F
6642026-03-04T01:35:29.726Z 4 040-049 32 32 32 3999 3999 3999 F F F
6652026-03-04T01:35:29.726ZMax gen: 32, Max flush: 3999
6662026-03-04T01:35:29.730Zdump test passed
6672026-03-04T01:35:29.733Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
6682026-03-04T01:35:29.748Z Extent 1
6692026-03-04T01:35:29.751ZGEN 32 32 32
6702026-03-04T01:35:29.751ZFLUSH_ID 3999 3999 3999
6712026-03-04T01:35:29.751ZDIRTY
6722026-03-04T01:35:29.751Z
6732026-03-04T01:35:29.751ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
6742026-03-04T01:35:29.751Z010 A A A A A A
6752026-03-04T01:35:29.753Z011 A A A A A A
6762026-03-04T01:35:29.753Z012 A A A A A A
6772026-03-04T01:35:29.756Z013 A A A A A A
6782026-03-04T01:35:29.756Z014 A A A A A A
6792026-03-04T01:35:29.760Z015 A A A A A A
6802026-03-04T01:35:29.760Z016 A A A A A A
6812026-03-04T01:35:29.763Z017 A A A A A A
6822026-03-04T01:35:29.763Z018 A A A A A A
6832026-03-04T01:35:29.765Z019 A A A A A A
6842026-03-04T01:35:29.768Zdump extent test passed
6852026-03-04T01:35:29.771Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
6862026-03-04T01:35:29.786ZExtent 2 Block in extent 0 Actual block 20
6872026-03-04T01:35:29.789Z
6882026-03-04T01:35:29.789Z DATA SHA256 VER
6892026-03-04T01:35:29.789Z------ ---------------------------------------------------------------- ---
6902026-03-04T01:35:29.789Z 0 2fcefb191db23102d9442568bbad40416516bcef9bf20218333c21b5ecf243b7 A
6912026-03-04T01:35:29.789Z 1 2fcefb191db23102d9442568bbad40416516bcef9bf20218333c21b5ecf243b7 A
6922026-03-04T01:35:29.792Z 2 2fcefb191db23102d9442568bbad40416516bcef9bf20218333c21b5ecf243b7 A
6932026-03-04T01:35:29.792Z
6942026-03-04T01:35:29.792ZNONCES 0 1 2 DIFF
6952026-03-04T01:35:29.792Z------ ------------------------ ------------------------ ------------------------ -----
6962026-03-04T01:35:29.792Z 0 43e5fd2a6f513b45786c96b2 43e5fd2a6f513b45786c96b2 43e5fd2a6f513b45786c96b2
6972026-03-04T01:35:29.792Z
6982026-03-04T01:35:29.792Z TAGS 0 1 2 DIFF
6992026-03-04T01:35:29.792Z------ -------------------------------- -------------------------------- -------------------------------- -----
7002026-03-04T01:35:29.792Z 0 622ed9968df740fc7e613894530f9e39 622ed9968df740fc7e613894530f9e39 622ed9968df740fc7e613894530f9e39
7012026-03-04T01:35:29.792Z
7022026-03-04T01:35:29.792ZHASHES 0 1 2 DIFF
7032026-03-04T01:35:29.792Z------ ---------------- ---------------- ---------------- -----
7042026-03-04T01:35:29.792Z
7052026-03-04T01:35:29.795Zdump block test passed
7062026-03-04T01:35:29.795ZInitial upstairs tests have completed, stopping all downstairs
7072026-03-04T01:35:34.824ZCreating 7 larger downstairs regions
7082026-03-04T01:35:35.186ZStarting 7 downstairs
7092026-03-04T01:35:40.192Zdsc restarted at PID: 1278
7102026-03-04T01:35:40.364ZNow do the replace-reconcile test
7112026-03-04T01:35:40.387ZUsing 8870 for the replacement port
7122026-03-04T01:39:05.292ZCompleted test: replace-reconcile
7132026-03-04T01:39:05.295ZNow do the replace-before-active test
7142026-03-04T01:45:13.599ZCompleted test: replace-before-active
7152026-03-04T01:45:13.599ZAll tests have completed, stopping all downstairs
7162026-03-04T01:45:13.628Z
7172026-03-04T01:45:13.628ZAll Tests have passed
7182026-03-04T01:45:13.628Z16:01 Test duration
7192026-03-04T01:45:13.633Z
7202026-03-04T01:45:13.633Zreal 16:00.454761921
7212026-03-04T01:45:13.633Zuser 40:35.843562094
7222026-03-04T01:45:13.633Zsys 31.726876638
7232026-03-04T01:45:13.633Ztrap 1.429050715
7242026-03-04T01:45:13.633Ztflt 0.082977526
7252026-03-04T01:45:13.633Zdflt 0.024175166
7262026-03-04T01:45:13.633Zkflt 0.001067404
7272026-03-04T01:45:13.633Zlock 24:09:31.340375035
7282026-03-04T01:45:13.633Zslp 1:14:03.195298575
7292026-03-04T01:45:13.633Zlat 4:50.748362047
7302026-03-04T01:45:13.633Zstop 0.071736216
7312026-03-04T01:45:13.633Z+ echo 'test-up-2region-encrypted ends'
7322026-03-04T01:45:13.633Ztest-up-2region-encrypted ends
7332026-03-04T01:45:18.631Zprocess exited: duration 971105 ms, exit code 0
7342026-03-04T01:45:18.631Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
7352026-03-04T01:46:18.669Zexec warning: : stdout descriptor may be held open by a background process; giving up!
7362026-03-04T01:46:18.670Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
7372026-03-04T01:46:18.681Zfound 17 output files
7382026-03-04T01:46:18.681Zuploading: /tmp/test_up-build/dsc-out.txt (55242 bytes)
7392026-03-04T01:46:19.695Zuploaded: /tmp/test_up-build/dsc-out.txt
7402026-03-04T01:46:19.695Zuploading: /tmp/test_up-build/test_up_out.txt (23037567 bytes)
7412026-03-04T01:46:20.927Zuploaded: /tmp/test_up-build/test_up_out.txt
7422026-03-04T01:46:20.930Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (551013 bytes)
7432026-03-04T01:46:21.953Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
7442026-03-04T01:46:21.956Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (548353 bytes)
7452026-03-04T01:46:22.977Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
7462026-03-04T01:46:22.981Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (979645 bytes)
7472026-03-04T01:46:24.007Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
7482026-03-04T01:46:24.011Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (543062 bytes)
7492026-03-04T01:46:25.036Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
7502026-03-04T01:46:25.039Zuploading: /tmp/test_up-build/dsc/downstairs-8850.txt (2481 bytes)
7512026-03-04T01:46:26.050Zuploaded: /tmp/test_up-build/dsc/downstairs-8850.txt
7522026-03-04T01:46:26.053Zuploading: /tmp/test_up-build/dsc/downstairs-8860.txt (42041 bytes)
7532026-03-04T01:46:27.065Zuploaded: /tmp/test_up-build/dsc/downstairs-8860.txt
7542026-03-04T01:46:27.068Zuploading: /tmp/test_up-build/dsc/downstairs-8870.txt (3796881 bytes)
7552026-03-04T01:46:28.115Zuploaded: /tmp/test_up-build/dsc/downstairs-8870.txt
7562026-03-04T01:46:28.118Zuploading: /tmp/debug/df.txt (1270 bytes)
7572026-03-04T01:46:29.125Zuploaded: /tmp/debug/df.txt
7582026-03-04T01:46:29.128Zuploading: /tmp/debug/dtrace.txt (1464854 bytes)
7592026-03-04T01:46:30.159Zuploaded: /tmp/debug/dtrace.txt
7602026-03-04T01:46:30.162Zuploading: /tmp/debug/iostat.txt (305992 bytes)
7612026-03-04T01:46:30.179Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 305992 -> 309568
7622026-03-04T01:46:31.188Zuploaded: /tmp/debug/iostat.txt
7632026-03-04T01:46:31.192Zuploading: /tmp/debug/mpstat.txt (745921 bytes)
7642026-03-04T01:46:31.211Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 745921 -> 755372
7652026-03-04T01:46:32.222Zuploaded: /tmp/debug/mpstat.txt
7662026-03-04T01:46:32.225Zuploading: /tmp/debug/paging.txt (132212 bytes)
7672026-03-04T01:46:32.235Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 132212 -> 133952
7682026-03-04T01:46:33.242Zuploaded: /tmp/debug/paging.txt
7692026-03-04T01:46:33.245Zuploading: /tmp/debug/prstat.txt (1480632 bytes)
7702026-03-04T01:46:33.270Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 1480632 -> 1498742
7712026-03-04T01:46:34.279Zuploaded: /tmp/debug/prstat.txt
7722026-03-04T01:46:34.282Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
7732026-03-04T01:46:35.291Zuploaded: /tmp/debug/psrinfo.txt
7742026-03-04T01:46:35.294Zuploading: /tmp/debug/upstairs-info.txt (233478 bytes)
7752026-03-04T01:46:36.314Zuploaded: /tmp/debug/upstairs-info.txt