01K86WXEW6SXS1RYY8DXPMMVFT: test-up-2region-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01K86WXY8WJN3S6VBFTB6N6GAR

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12025-10-22T21:42:44.125Zjob dependencies complete; ready to run (waiting for 18 m 24 s)
22025-10-22T21:43:48.459Zjob assigned to worker 01K86XZW6S6HABWSWXH7785A6V [factory aws, i-0ff6f76c96ad51609] (queued for 1 m 4 s)
32025-10-22T21:43:56.768Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42025-10-22T21:43:58.452Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52025-10-22T21:43:58.452Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62025-10-22T21:44:00.108Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72025-10-22T21:44:00.114Zdownloading input: /input/build/work/bins/crutest.gz
82025-10-22T21:44:01.531Zdownloaded input: /input/build/work/bins/crutest.gz
92025-10-22T21:44:01.532Zdownloading input: /input/build/work/bins/dsc.gz
102025-10-22T21:44:02.236Zdownloaded input: /input/build/work/bins/dsc.gz
112025-10-22T21:44:02.240Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122025-10-22T21:44:02.260Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132025-10-22T21:44:02.260Zdownloading input: /input/build/work/scripts/test_ds.sh
142025-10-22T21:44:02.281Zdownloaded input: /input/build/work/scripts/test_ds.sh
152025-10-22T21:44:02.282Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162025-10-22T21:44:02.305Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172025-10-22T21:44:02.306Zdownloading input: /input/build/work/scripts/test_repair.sh
182025-10-22T21:44:02.329Zdownloaded input: /input/build/work/scripts/test_repair.sh
192025-10-22T21:44:02.329Zdownloading input: /input/build/work/scripts/test_replay.sh
202025-10-22T21:44:02.352Zdownloaded input: /input/build/work/scripts/test_replay.sh
212025-10-22T21:44:02.352Zdownloading input: /input/build/work/scripts/test_up.sh
222025-10-22T21:44:02.374Zdownloaded input: /input/build/work/scripts/test_up.sh
232025-10-22T21:44:02.374Zdownloading input: /input/build/work/scripts/upstairs_info.d
242025-10-22T21:44:02.401Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252025-10-22T21:44:02.405Zdownloading input: /input/build/tmp/cargo-test-out.log
262025-10-22T21:44:03.059Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272025-10-22T21:44:03.060Zstarting task 0: "setup"
282025-10-22T21:44:03.066Z++ uname -s
292025-10-22T21:44:03.066Z+ kern=SunOS
302025-10-22T21:44:03.066Z+ build_user=build
312025-10-22T21:44:03.066Z+ build_uid=12345
322025-10-22T21:44:03.066Z+ work_dir=/work
332025-10-22T21:44:03.066Z+ input_dir=/input
342025-10-22T21:44:03.066Z+ [[ 0 == 12345 ]]
352025-10-22T21:44:03.066Z+ case "$kern" in
362025-10-22T21:44:03.066Z+ groupadd -g 12345 build
372025-10-22T21:44:03.066Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382025-10-22T21:44:05.071Z+ zfs create -o mountpoint=/work rpool/work
392025-10-22T21:44:05.393Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402025-10-22T21:44:05.398Z+ home_fs=zfs
412025-10-22T21:44:05.398Z+ [[ zfs == autofs ]]
422025-10-22T21:44:05.398Z+ mkdir -p /home/build
432025-10-22T21:44:05.401Z+ chown build:build /home/build /work
442025-10-22T21:44:06.399Z+ chmod 0700 /home/build /work
452025-10-22T21:44:06.403Zprocess exited: duration 3349 ms, exit code 0
 
462025-10-22T21:44:06.409Zstarting task 1: "authentication"
472025-10-22T21:44:06.429Zprocess exited: duration 20 ms, exit code 0
 
482025-10-22T21:44:06.436Zstarting task 2: "build"
492025-10-22T21:44:06.441Z+ banner cores
502025-10-22T21:44:06.444Z
512025-10-22T21:44:06.444Z #### #### ##### ###### ####
522025-10-22T21:44:06.444Z # # # # # # # #
532025-10-22T21:44:06.444Z # # # # # ##### ####
542025-10-22T21:44:06.444Z # # # ##### # #
552025-10-22T21:44:06.444Z # # # # # # # # #
562025-10-22T21:44:06.444Z #### #### # # ###### ####
572025-10-22T21:44:06.445Z
582025-10-22T21:44:06.445Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
592025-10-22T21:44:06.453Z+ banner unpack
602025-10-22T21:44:06.456Z
612025-10-22T21:44:06.456Z # # # # ##### ## #### # #
622025-10-22T21:44:06.456Z # # ## # # # # # # # # #
632025-10-22T21:44:06.456Z # # # # # # # # # # ####
642025-10-22T21:44:06.456Z # # # # # ##### ###### # # #
652025-10-22T21:44:06.456Z # # # ## # # # # # # #
662025-10-22T21:44:06.456Z #### # # # # # #### # #
672025-10-22T21:44:06.456Z
682025-10-22T21:44:06.456Z+ mkdir -p /var/tmp/bins
692025-10-22T21:44:06.460Z+ for t in "$input/bins/"*.gz
702025-10-22T21:44:06.460Z++ basename /input/build/work/bins/crucible-downstairs.gz
712025-10-22T21:44:06.460Z+ b=crucible-downstairs.gz
722025-10-22T21:44:06.460Z+ b=crucible-downstairs
732025-10-22T21:44:06.460Z+ gunzip
742025-10-22T21:44:09.395Z+ chmod +x /var/tmp/bins/crucible-downstairs
752025-10-22T21:44:09.398Z+ for t in "$input/bins/"*.gz
762025-10-22T21:44:09.398Z++ basename /input/build/work/bins/crucible-hammer.gz
772025-10-22T21:44:09.402Z+ b=crucible-hammer.gz
782025-10-22T21:44:09.402Z+ b=crucible-hammer
792025-10-22T21:44:09.402Z+ gunzip
802025-10-22T21:44:12.193Z+ chmod +x /var/tmp/bins/crucible-hammer
812025-10-22T21:44:12.196Z+ for t in "$input/bins/"*.gz
822025-10-22T21:44:12.196Z++ basename /input/build/work/bins/crutest.gz
832025-10-22T21:44:12.199Z+ b=crutest.gz
842025-10-22T21:44:12.199Z+ b=crutest
852025-10-22T21:44:12.199Z+ gunzip
862025-10-22T21:44:15.115Z+ chmod +x /var/tmp/bins/crutest
872025-10-22T21:44:15.118Z+ for t in "$input/bins/"*.gz
882025-10-22T21:44:15.118Z++ basename /input/build/work/bins/dsc.gz
892025-10-22T21:44:15.121Z+ b=dsc.gz
902025-10-22T21:44:15.121Z+ b=dsc
912025-10-22T21:44:15.122Z+ gunzip
922025-10-22T21:44:16.498Z+ chmod +x /var/tmp/bins/dsc
932025-10-22T21:44:16.501Z+ export BINDIR=/var/tmp/bins
942025-10-22T21:44:16.501Z+ BINDIR=/var/tmp/bins
952025-10-22T21:44:16.502Z+ jobpid=1155
962025-10-22T21:44:16.502Z+ echo 'Setup debug logging'
972025-10-22T21:44:16.502ZSetup debug logging
982025-10-22T21:44:16.502Z+ mkdir /tmp/debug
992025-10-22T21:44:16.502Z+ sleep 3600
1002025-10-22T21:44:16.505Z+ psrinfo -v
1012025-10-22T21:44:16.509Z+ df -h
1022025-10-22T21:44:16.514Z+ prstat -d d -mLc 1
1032025-10-22T21:44:16.517Z+ iostat -T d -xn 1
1042025-10-22T21:44:16.517Z+ mpstat -T d 1
1052025-10-22T21:44:16.517Z+ vmstat -T d -p 1
1062025-10-22T21:44:16.517Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1072025-10-22T21:44:16.517Z+ banner test_up_2r_encrypted
1082025-10-22T21:44:16.521Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1092025-10-22T21:44:16.521Z #####
1102025-10-22T21:44:16.521Z ##### ###### #### ##### # # ##### # # #####
1112025-10-22T21:44:16.521Z # # # # # # # # # # #
1122025-10-22T21:44:16.521Z # ##### #### # # # # # ##### # #
1132025-10-22T21:44:16.521Z # # # # # # ##### # #####
1142025-10-22T21:44:16.521Z # # # # # # # # # # #
1152025-10-22T21:44:16.521Z # ###### #### # ####### #### # ####### ####### # #
1162025-10-22T21:44:16.521Z
1172025-10-22T21:44:16.521Z+ ptime -m bash /input/build/work/scripts/test_up.sh -r 2 -N encrypted
1182025-10-22T21:44:16.531Z/input/build/work
1192025-10-22T21:44:16.542ZUsing 2 region sets
1202025-10-22T21:44:16.542ZTurn off color for downstairs dump
1212025-10-22T21:44:16.558ZUpstairs using key: Mh5+cDXnjTwiXFBStt7r54PzNsgmw6A08uZqaayXxAI=
1222025-10-22T21:44:16.562Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1232025-10-22T21:44:16.562ZCreating 6 downstairs regions
1242025-10-22T21:44:16.964ZStarting 6 downstairs
1252025-10-22T21:44:16.967Zdsc started at PID: 1203
1262025-10-22T21:44:22.317ZDisable automatic restart on all downstairs
1272025-10-22T21:44:22.372Z
1282025-10-22T21:44:22.372ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1292025-10-22T21:44:22.375ZRunning test: span
1302025-10-22T21:44:26.564ZCompleted test: span
1312025-10-22T21:44:31.572ZRunning test: big
1322025-10-22T21:44:36.156ZCompleted test: big
1332025-10-22T21:44:41.164ZRunning test: dep
1342025-10-22T21:45:15.051ZCompleted test: dep
1352025-10-22T21:45:20.058ZRunning test: balloon
1362025-10-22T21:49:35.558ZCompleted test: balloon
1372025-10-22T21:49:40.566ZRunning test: deactivate
1382025-10-22T21:49:45.242ZCompleted test: deactivate
1392025-10-22T21:49:50.250ZRunning hammer
1402025-10-22T21:50:08.341ZRun repair tests
1412025-10-22T21:50:08.345Z/var/tmp/bins/crutest fill -g 31 -q --key Mh5+cDXnjTwiXFBStt7r54PzNsgmw6A08uZqaayXxAI= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1422025-10-22T21:50:08.404Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.17263244Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"task":"crutest"}
1432025-10-22T21:50:08.408Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.1732379Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"task":"crutest"}
1442025-10-22T21:50:08.408Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.175176749Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"task":"crutest"}
1452025-10-22T21:50:08.408Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.175393903Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246}
1462025-10-22T21:50:08.408Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9063dde9565406ad9efc6e457bbffa920b2c2351\",\n git_commit_timestamp: \"2025-10-22T21:23:18.000000000Z\",\n git_branch: \"mkeeter/check-hashes\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.175444095Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246}
1472025-10-22T21:50:08.408Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.175467741Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246}
1482025-10-22T21:50:08.408Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.175506124Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246}
1492025-10-22T21:50:08.408Z{"msg":"Crucible 29a6f222-d8d9-4671-b4c0-bf5c8006ff78 has session id: b2ddddd9-6ca6-462a-a8e2-7a22f81592e5","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.176113814Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1502025-10-22T21:50:08.408Z{"msg":"Upstairs opts: Upstairs UUID: 29a6f222-d8d9-4671-b4c0-bf5c8006ff78, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.176175963Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1512025-10-22T21:50:08.408Z{"msg":"Crucible stats registered with UUID: 29a6f222-d8d9-4671-b4c0-bf5c8006ff78","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.176214707Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1522025-10-22T21:50:08.408Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-10-22T21:50:07.176233514Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1532025-10-22T21:50:08.408Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.17638005Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1542025-10-22T21:50:08.408Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.176432531Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1552025-10-22T21:50:08.409Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.176452608Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1562025-10-22T21:50:08.413Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.178327307Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"task":"crutest"}
1572025-10-22T21:50:08.413Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.178425431Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246}
1582025-10-22T21:50:08.413Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9063dde9565406ad9efc6e457bbffa920b2c2351\",\n git_commit_timestamp: \"2025-10-22T21:23:18.000000000Z\",\n git_branch: \"mkeeter/check-hashes\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.178474313Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246}
1592025-10-22T21:50:08.413Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.178496709Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246}
1602025-10-22T21:50:08.413Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.178519025Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246}
1612025-10-22T21:50:08.413Z{"msg":"Crucible 29a6f222-d8d9-4671-b4c0-bf5c8006ff78 has session id: 2d9aea10-29d4-4c27-bde6-c3a3a074399b","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.178980049Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
1622025-10-22T21:50:08.413Z{"msg":"Upstairs opts: Upstairs UUID: 29a6f222-d8d9-4671-b4c0-bf5c8006ff78, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.179048118Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
1632025-10-22T21:50:08.413Z{"msg":"Crucible stats registered with UUID: 29a6f222-d8d9-4671-b4c0-bf5c8006ff78","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.179087671Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
1642025-10-22T21:50:08.413Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-10-22T21:50:07.179130714Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
1652025-10-22T21:50:08.413Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.17915842Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
1662025-10-22T21:50:08.414Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.179190134Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
1672025-10-22T21:50:08.414Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.179208401Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
1682025-10-22T21:50:08.414Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.179951718Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"task":"crutest"}
1692025-10-22T21:50:08.414Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.179974644Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246}
1702025-10-22T21:50:08.414Z{"msg":"29a6f222-d8d9-4671-b4c0-bf5c8006ff78 active request set","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.180093365Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1712025-10-22T21:50:08.414Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.180162943Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"client":"0","":"downstairs","session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1722025-10-22T21:50:08.414Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.180212215Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"client":"1","":"downstairs","session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1732025-10-22T21:50:08.414Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.180252298Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"client":"2","":"downstairs","session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1742025-10-22T21:50:08.414Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.180279354Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1752025-10-22T21:50:08.414Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.180305419Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1762025-10-22T21:50:08.414Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.180549459Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1772025-10-22T21:50:08.414Z{"msg":"ds_connection connected from Ok(127.0.0.1:65347)","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.180613639Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1782025-10-22T21:50:08.414Z{"msg":"ds_connection connected from Ok(127.0.0.1:36847)","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.180655892Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1792025-10-22T21:50:08.414Z{"msg":"ds_connection connected from Ok(127.0.0.1:40445)","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.180719331Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1802025-10-22T21:50:08.419Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.181733763Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"client":"1","":"downstairs","session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1812025-10-22T21:50:08.419Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.181796143Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"client":"2","":"downstairs","session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1822025-10-22T21:50:08.419Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.181841985Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"client":"0","":"downstairs","session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1832025-10-22T21:50:08.419Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.182160622Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1842025-10-22T21:50:08.419Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.182196246Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1852025-10-22T21:50:08.419Z{"msg":"[0]R flush_numbers: [4003, 4003, 4003, 4003, 4003]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.182214123Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"downstairs","":"downstairs","session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1862025-10-22T21:50:08.419Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.182238869Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"downstairs","":"downstairs","session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1872025-10-22T21:50:08.419Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.182256436Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"downstairs","":"downstairs","session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1882025-10-22T21:50:08.419Z{"msg":"[1]R flush_numbers: [4003, 4003, 4003, 4003, 4003]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.18229814Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"downstairs","":"downstairs","session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1892025-10-22T21:50:08.419Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.182315367Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"downstairs","":"downstairs","session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1902025-10-22T21:50:08.419Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.182339722Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"downstairs","":"downstairs","session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1912025-10-22T21:50:08.419Z{"msg":"[2]R flush_numbers: [4003, 4003, 4003, 4003, 4003]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.18235702Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"downstairs","":"downstairs","session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1922025-10-22T21:50:08.419Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.182380386Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"downstairs","":"downstairs","session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1932025-10-22T21:50:08.419Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.182398093Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"downstairs","":"downstairs","session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1942025-10-22T21:50:08.419Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.182420569Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"downstairs","":"downstairs","session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1952025-10-22T21:50:08.420Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.182437317Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"downstairs","":"downstairs","session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1962025-10-22T21:50:08.420Z{"msg":"Next flush: 4004","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.182460073Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"downstairs","":"downstairs","session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1972025-10-22T21:50:08.420Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.18247646Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"downstairs","":"downstairs","session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1982025-10-22T21:50:08.420Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.182499796Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
1992025-10-22T21:50:08.420Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.182515353Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"client":"0","":"downstairs","session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
2002025-10-22T21:50:08.420Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.182547178Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"client":"1","":"downstairs","session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
2012025-10-22T21:50:08.420Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.18259873Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"client":"2","":"downstairs","session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
2022025-10-22T21:50:08.420Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.182634834Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
2032025-10-22T21:50:08.420Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.182834331Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
2042025-10-22T21:50:08.420Z{"msg":"29a6f222-d8d9-4671-b4c0-bf5c8006ff78 is now active with session: b2ddddd9-6ca6-462a-a8e2-7a22f81592e5","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.182942043Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
2052025-10-22T21:50:08.420Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.183004712Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"session_id":"b2ddddd9-6ca6-462a-a8e2-7a22f81592e5"}
2062025-10-22T21:50:08.420Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.183050055Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246}
2072025-10-22T21:50:08.420Z{"msg":"Activated sub_volume 29a6f222-d8d9-4671-b4c0-bf5c8006ff78","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.183075941Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246}
2082025-10-22T21:50:08.420Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.183103536Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246}
2092025-10-22T21:50:08.420Z{"msg":"29a6f222-d8d9-4671-b4c0-bf5c8006ff78 active request set","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.183119193Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2102025-10-22T21:50:08.420Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.18313996Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"client":"0","":"downstairs","session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2112025-10-22T21:50:08.420Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.183170535Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"client":"1","":"downstairs","session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2122025-10-22T21:50:08.420Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.183217447Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"client":"2","":"downstairs","session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2132025-10-22T21:50:08.420Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.18325914Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2142025-10-22T21:50:08.420Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.183300863Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2152025-10-22T21:50:08.420Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.183337437Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2162025-10-22T21:50:08.420Z{"msg":"ds_connection connected from Ok(127.0.0.1:60860)","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.183356774Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2172025-10-22T21:50:08.420Z{"msg":"ds_connection connected from Ok(127.0.0.1:57428)","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.183394068Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2182025-10-22T21:50:08.420Z{"msg":"ds_connection connected from Ok(127.0.0.1:54984)","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.183423483Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2192025-10-22T21:50:08.420Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.183704067Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"client":"1","":"downstairs","session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2202025-10-22T21:50:08.420Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.183760727Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"client":"0","":"downstairs","session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2212025-10-22T21:50:08.420Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.183816498Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"client":"2","":"downstairs","session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2222025-10-22T21:50:08.420Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.18404384Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2232025-10-22T21:50:08.420Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.184144274Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2242025-10-22T21:50:08.420Z{"msg":"[0]R flush_numbers: [5318, 5317, 5316, 5314, 5318]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.184183347Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"downstairs","":"downstairs","session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2252025-10-22T21:50:08.420Z{"msg":"[0]R generation: [14, 14, 14, 13, 14]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.184202954Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"downstairs","":"downstairs","session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2262025-10-22T21:50:08.420Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.18422927Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"downstairs","":"downstairs","session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2272025-10-22T21:50:08.420Z{"msg":"[1]R flush_numbers: [5318, 5317, 5316, 5314, 5318]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.184247357Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"downstairs","":"downstairs","session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2282025-10-22T21:50:08.420Z{"msg":"[1]R generation: [14, 14, 14, 13, 14]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.184362058Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V",Wait for a query_work_queue command to finish before sending IO
2292025-10-22T21:50:08.420Z"pid":1246,"":"downstairs","":"downstairs","session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2302025-10-22T21:50:08.420Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.18440634Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"downstairs","":"downstairs","session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2312025-10-22T21:50:08.420Z{"msg":"[2]R flush_numbers: [5318, 5317, 5316, 5314, 5318]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.184426317Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"downstairs","":"downstairs","session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2322025-10-22T21:50:08.421Z{"msg":"[2]R generation: [14, 14, 14, 13, 14]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.184443984Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"downstairs","":"downstairs","session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2332025-10-22T21:50:08.421Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.18446998Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"downstairs","":"downstairs","session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2342025-10-22T21:50:08.421Z{"msg":"Max found gen is 15","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.184486857Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"downstairs","":"downstairs","session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2352025-10-22T21:50:08.421Z{"msg":"Generation requested: 31 >= found:15","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.184515862Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"downstairs","":"downstairs","session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2362025-10-22T21:50:08.421Z{"msg":"Next flush: 5319","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.184532359Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"downstairs","":"downstairs","session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2372025-10-22T21:50:08.421Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.184548307Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"":"downstairs","":"downstairs","session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2382025-10-22T21:50:08.421Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.184571943Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2392025-10-22T21:50:08.421Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.184595989Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"client":"0","":"downstairs","session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2402025-10-22T21:50:08.421Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.184628163Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"client":"1","":"downstairs","session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2412025-10-22T21:50:08.421Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.184653619Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"client":"2","":"downstairs","session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2422025-10-22T21:50:08.421Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.184677625Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2432025-10-22T21:50:08.421Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.184693863Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2442025-10-22T21:50:08.421Z{"msg":"29a6f222-d8d9-4671-b4c0-bf5c8006ff78 is now active with session: 2d9aea10-29d4-4c27-bde6-c3a3a074399b","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.184718808Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2452025-10-22T21:50:08.421Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.184734086Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246,"session_id":"2d9aea10-29d4-4c27-bde6-c3a3a074399b"}
2462025-10-22T21:50:08.421Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.184756192Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246}
2472025-10-22T21:50:08.421Z{"msg":"Activated sub_volume 29a6f222-d8d9-4671-b4c0-bf5c8006ff78","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:07.184778529Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246}
2482025-10-22T21:50:08.421ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
2492025-10-22T21:50:08.421ZFill test
2502025-10-22T21:50:08.460ZRead and Verify all blocks (0..100 range:false)
2512025-10-22T21:50:08.540ZWrote out file "/var/tmp/test_up-build/verify_file"
2522025-10-22T21:50:08.540ZCLIENT: Tests done. All submitted work has been ACK'd
2532025-10-22T21:50:08.544Z----------------------------------------------------------------
2542025-10-22T21:50:08.544Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2552025-10-22T21:50:08.544ZStates: Active Active Active
2562025-10-22T21:50:08.544ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2572025-10-22T21:50:08.544Z 1002 Acked Read 50 Done Done Done false
2582025-10-22T21:50:08.544Z STATES DS:0 DS:1 DS:2 TOTAL
2592025-10-22T21:50:08.544Z Sent 0 0 0 0
2602025-10-22T21:50:08.544Z Done 1 1 1 3
2612025-10-22T21:50:08.544Z Skipped 0 0 0 0
2622025-10-22T21:50:08.544Z Error 0 0 0 0
2632025-10-22T21:50:08.544ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2642025-10-22T21:50:08.544ZDownstairs last five completed: 1001 1000
2652025-10-22T21:50:08.544ZUpstairs last five completed: 1002 1001 1000
2662025-10-22T21:50:08.544Z----------------------------------------------------------------
2672025-10-22T21:50:08.544Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2682025-10-22T21:50:08.544ZStates: Active Active Active
2692025-10-22T21:50:08.544ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2702025-10-22T21:50:08.544Z 1002 Acked Read 50 Sent Done Sent false
2712025-10-22T21:50:08.544Z STATES DS:0 DS:1 DS:2 TOTAL
2722025-10-22T21:50:08.544Z Sent 1 0 1 2
2732025-10-22T21:50:08.544Z Done 0 1 0 1
2742025-10-22T21:50:08.544Z Skipped 0 0 0 0
2752025-10-22T21:50:08.544Z Error 0 0 0 0
2762025-10-22T21:50:08.544ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2772025-10-22T21:50:08.544ZDownstairs last five completed: 1001 1000
2782025-10-22T21:50:08.544ZUpstairs last five completed: 1002 1001 1000
2792025-10-22T21:50:08.544ZCLIENT: Up:0 ds:2 act:6
2802025-10-22T21:50:12.541Z----------------------------------------------------------------
2812025-10-22T21:50:12.542Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2822025-10-22T21:50:12.542ZDownstairs last five completed: 1003 1002 1001 1000
2832025-10-22T21:50:12.542ZUpstairs last five completed: 1003 1002 1001 1000
2842025-10-22T21:50:12.545Z----------------------------------------------------------------
2852025-10-22T21:50:12.545Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2862025-10-22T21:50:12.545ZDownstairs last five completed: 1003 1002 1001 1000
2872025-10-22T21:50:12.545ZUpstairs last five completed: 1003 1002 1001 1000
2882025-10-22T21:50:12.545ZCLIENT: Up:0 ds:0 act:6
2892025-10-22T21:50:12.545ZCLIENT: All crucible jobs finished, exiting program
2902025-10-22T21:50:12.545Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-10-22T21:50:11.309569416Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1246}
2912025-10-22T21:50:12.553ZRepair setup passed
2922025-10-22T21:50:12.557ZCopy the region for /var/tmp/test_up-build/8830
2932025-10-22T21:50:12.560Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2942025-10-22T21:50:12.563Z/var/tmp/bins/crutest fill -g 32 -q --key Mh5+cDXnjTwiXFBStt7r54PzNsgmw6A08uZqaayXxAI= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2952025-10-22T21:50:12.619Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.38658572Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"task":"crutest"}
2962025-10-22T21:50:12.623Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.387158965Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"task":"crutest"}
2972025-10-22T21:50:12.623Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.388953338Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"task":"crutest"}
2982025-10-22T21:50:12.623Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.389165433Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252}
2992025-10-22T21:50:12.623Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9063dde9565406ad9efc6e457bbffa920b2c2351\",\n git_commit_timestamp: \"2025-10-22T21:23:18.000000000Z\",\n git_branch: \"mkeeter/check-hashes\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.389214884Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252}
3002025-10-22T21:50:12.623Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.389238481Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252}
3012025-10-22T21:50:12.623Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.389261397Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252}
3022025-10-22T21:50:12.623Z{"msg":"Crucible e160138e-5d13-45a3-971a-913411fb444c has session id: a69db812-48ce-48c0-aaff-608925fe6823","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.389939954Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3032025-10-22T21:50:12.623Z{"msg":"Upstairs opts: Upstairs UUID: e160138e-5d13-45a3-971a-913411fb444c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.389987636Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3042025-10-22T21:50:12.623Z{"msg":"Crucible stats registered with UUID: e160138e-5d13-45a3-971a-913411fb444c","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.390007563Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3052025-10-22T21:50:12.623Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-10-22T21:50:11.390031739Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3062025-10-22T21:50:12.623Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.390193552Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3072025-10-22T21:50:12.623Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.390243374Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3082025-10-22T21:50:12.623Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.390263361Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3092025-10-22T21:50:12.627Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.392237144Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"task":"crutest"}
3102025-10-22T21:50:12.627Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.392327949Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252}
3112025-10-22T21:50:12.628Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9063dde9565406ad9efc6e457bbffa920b2c2351\",\n git_commit_timestamp: \"2025-10-22T21:23:18.000000000Z\",\n git_branch: \"mkeeter/check-hashes\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.392386419Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252}
3122025-10-22T21:50:12.628Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.392413315Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252}
3132025-10-22T21:50:12.628Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.39244074Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252}
3142025-10-22T21:50:12.628Z{"msg":"Crucible e160138e-5d13-45a3-971a-913411fb444c has session id: fbae622d-ff44-439b-9aed-dd2911e65d55","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.392769456Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3152025-10-22T21:50:12.628Z{"msg":"Upstairs opts: Upstairs UUID: e160138e-5d13-45a3-971a-913411fb444c, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.392835965Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3162025-10-22T21:50:12.628Z{"msg":"Crucible stats registered with UUID: e160138e-5d13-45a3-971a-913411fb444c","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.392884826Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3172025-10-22T21:50:12.628Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-10-22T21:50:11.392910032Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3182025-10-22T21:50:12.628Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.392944727Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3192025-10-22T21:50:12.628Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.392964413Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3202025-10-22T21:50:12.628Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.392988979Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3212025-10-22T21:50:12.628Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.393577212Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"task":"crutest"}
3222025-10-22T21:50:12.628Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.393628463Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252}
3232025-10-22T21:50:12.628Z{"msg":"e160138e-5d13-45a3-971a-913411fb444c active request set","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.393714779Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3242025-10-22T21:50:12.628Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.393794446Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"client":"0","":"downstairs","session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3252025-10-22T21:50:12.628Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.393853386Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"client":"1","":"downstairs","session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3262025-10-22T21:50:12.628Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.393884101Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"client":"2","":"downstairs","session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3272025-10-22T21:50:12.628Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.393927944Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3282025-10-22T21:50:12.628Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.393963708Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3292025-10-22T21:50:12.628Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.393991963Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3302025-10-22T21:50:12.628Z{"msg":"ds_connection connected from Ok(127.0.0.1:55967)","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.394018779Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3312025-10-22T21:50:12.628Z{"msg":"ds_connection connected from Ok(127.0.0.1:53431)","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.394044765Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3322025-10-22T21:50:12.628Z{"msg":"ds_connection connected from Ok(127.0.0.1:49415)","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.39407302Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3332025-10-22T21:50:12.634Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.395326062Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"client":"1","":"downstairs","session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3342025-10-22T21:50:12.634Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.395392351Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"client":"2","":"downstairs","session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3352025-10-22T21:50:12.634Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.395455191Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"client":"0","":"downstairs","session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3362025-10-22T21:50:12.634Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.395771379Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3372025-10-22T21:50:12.634Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.395808492Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3382025-10-22T21:50:12.634Z{"msg":"[0]R flush_numbers: [4004, 4004, 4004, 4004, 4004]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.395826869Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"downstairs","":"downstairs","session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3392025-10-22T21:50:12.634Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.395856304Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"downstairs","":"downstairs","session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3402025-10-22T21:50:12.634Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.395877661Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"downstairs","":"downstairs","session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3412025-10-22T21:50:12.634Z{"msg":"[1]R flush_numbers: [4004, 4004, 4004, 4004, 4004]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.395895338Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"downstairs","":"downstairs","session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3422025-10-22T21:50:12.634Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.395919724Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"downstairs","":"downstairs","session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3432025-10-22T21:50:12.634Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.395937311Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"downstairs","":"downstairs","session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3442025-10-22T21:50:12.634Z{"msg":"[2]R flush_numbers: [4004, 4004, 4004, 4004, 4004]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.395960827Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"downstairs","":"downstairs","session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3452025-10-22T21:50:12.634Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.395978024Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"downstairs","":"downstairs","session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3462025-10-22T21:50:12.634Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.39600214Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"downstairs","":"downstairs","session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3472025-10-22T21:50:12.634Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.396018638Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"downstairs","":"downstairs","session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3482025-10-22T21:50:12.634Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.396057071Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"downstairs","":"downstairs","session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3492025-10-22T21:50:12.634Z{"msg":"Next flush: 4005","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.396083497Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"downstairs","":"downstairs","session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3502025-10-22T21:50:12.634Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.39612233Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"downstairs","":"downstairs","session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3512025-10-22T21:50:12.634Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.396155465Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3522025-10-22T21:50:12.634Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.39618649Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"client":"0","":"downstairs","session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3532025-10-22T21:50:12.634Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.396214735Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"client":"1","":"downstairs","session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3542025-10-22T21:50:12.634Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.39624296Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"client":"2","":"downstairs","session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3552025-10-22T21:50:12.634Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.396271306Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3562025-10-22T21:50:12.634Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.396471543Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3572025-10-22T21:50:12.634Z{"msg":"e160138e-5d13-45a3-971a-913411fb444c is now active with session: a69db812-48ce-48c0-aaff-608925fe6823","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.396535022Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3582025-10-22T21:50:12.634Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.396576205Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"session_id":"a69db812-48ce-48c0-aaff-608925fe6823"}
3592025-10-22T21:50:12.634Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.396595482Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252}
3602025-10-22T21:50:12.634Z{"msg":"Activated sub_volume e160138e-5d13-45a3-971a-913411fb444c","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.396621238Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252}
3612025-10-22T21:50:12.634Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.396639035Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252}
3622025-10-22T21:50:12.634Z{"msg":"e160138e-5d13-45a3-971a-913411fb444c active request set","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.396654072Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3632025-10-22T21:50:12.634Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.396678828Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"client":"0","":"downstairs","session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3642025-10-22T21:50:12.635Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.39672447Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"client":"1","":"downstairs","session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3652025-10-22T21:50:12.635Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.396769173Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"client":"2","":"downstairs","session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3662025-10-22T21:50:12.635Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.396804407Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3672025-10-22T21:50:12.635Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.396864507Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3682025-10-22T21:50:12.635Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.396903411Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3692025-10-22T21:50:12.635Z{"msg":"ds_connection connected from Ok(127.0.0.1:36333)","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.396935546Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3702025-10-22T21:50:12.635Z{"msg":"ds_connection connected from Ok(127.0.0.1:60769)","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.396961401Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3712025-10-22T21:50:12.635Z{"msg":"ds_connection connected from Ok(127.0.0.1:45641)","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.396989947Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3722025-10-22T21:50:12.635Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.397229667Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"client":"1","":"downstairs","session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3732025-10-22T21:50:12.635Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.397295726Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"client":"2","":"downstairs","session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3742025-10-22T21:50:12.635Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.397353197Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"client":"0","":"downstairs","session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3752025-10-22T21:50:12.635Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.397587328Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3762025-10-22T21:50:12.635Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.397675863Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3772025-10-22T21:50:12.635Z{"msg":"[0]R flush_numbers: [5319, 5319, 5319, 5319, 5319]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.397725345Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"downstairs","":"downstairs","session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3782025-10-22T21:50:12.635Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.397751431Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"downstairs","":"downstairs","session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3792025-10-22T21:50:12.635Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.397770517Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"downstairs","":"downstairs","session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3802025-10-22T21:50:12.636Z{"msg":"[1]R flush_numbers: [5319, 5319, 5319, 5319, 5319]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.397794963Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"downstairs","":"downstairs","session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3812025-10-22T21:50:12.636Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.3978131Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"downstairs","":"downstairs","session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3822025-10-22T21:50:12.636Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.397830617Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"downstairs","":"downstairs","session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3832025-10-22T21:50:12.636Z{"msg":"[2]R flush_numbers: [5319, 5319, 5319, 5319, 5319]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.397854274Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"downstairs","":"downstairs","session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3842025-10-22T21:50:12.636Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.397871461Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"downstairs","":"downstairs","session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3852025-10-22T21:50:12.636Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.397895107Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"downstairs","":"downstairs","session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3862025-10-22T21:50:12.636Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.397911474Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO
3872025-10-22T21:50:12.636Zdownstairs","session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3882025-10-22T21:50:12.636Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.397943089Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"downstairs","":"downstairs","session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3892025-10-22T21:50:12.636Z{"msg":"Next flush: 5320","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.397960096Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"downstairs","":"downstairs","session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3902025-10-22T21:50:12.636Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.397982622Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"":"downstairs","":"downstairs","session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3912025-10-22T21:50:12.636Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.398000289Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3922025-10-22T21:50:12.636Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.398016467Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"client":"0","":"downstairs","session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3932025-10-22T21:50:12.636Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.398039733Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"client":"1","":"downstairs","session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3942025-10-22T21:50:12.636Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.39805655Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"client":"2","":"downstairs","session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3952025-10-22T21:50:12.636Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.398080086Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3962025-10-22T21:50:12.636Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.398096833Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3972025-10-22T21:50:12.636Z{"msg":"e160138e-5d13-45a3-971a-913411fb444c is now active with session: fbae622d-ff44-439b-9aed-dd2911e65d55","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.39811476Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3982025-10-22T21:50:12.636Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.398136527Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252,"session_id":"fbae622d-ff44-439b-9aed-dd2911e65d55"}
3992025-10-22T21:50:12.636Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.398152874Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252}
4002025-10-22T21:50:12.636Z{"msg":"Activated sub_volume e160138e-5d13-45a3-971a-913411fb444c","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:11.398174081Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1252}
4012025-10-22T21:50:12.636ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
4022025-10-22T21:50:12.636ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4032025-10-22T21:50:12.636ZFill test
4042025-10-22T21:50:12.673ZRead and Verify all blocks (0..100 range:false)
4052025-10-22T21:50:12.753ZWrote out file "/var/tmp/test_up-build/verify_file"
4062025-10-22T21:50:12.753ZCLIENT: Tests done. All submitted work has been ACK'd
4072025-10-22T21:50:12.753Z----------------------------------------------------------------
4082025-10-22T21:50:12.753Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4092025-10-22T21:50:12.753ZStates: Active Active Active
4102025-10-22T21:50:12.753ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4112025-10-22T21:50:12.753Z 1002 Acked Read 50 Done Done Done false
4122025-10-22T21:50:12.753Z STATES DS:0 DS:1 DS:2 TOTAL
4132025-10-22T21:50:12.753Z Sent 0 0 0 0
4142025-10-22T21:50:12.753Z Done 1 1 1 3
4152025-10-22T21:50:12.753Z Skipped 0 0 0 0
4162025-10-22T21:50:12.753Z Error 0 0 0 0
4172025-10-22T21:50:12.753ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4182025-10-22T21:50:12.753ZDownstairs last five completed: 1001 1000
4192025-10-22T21:50:12.753ZUpstairs last five completed: 1002 1001 1000
4202025-10-22T21:50:12.756Z----------------------------------------------------------------
4212025-10-22T21:50:12.757Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4222025-10-22T21:50:12.757ZStates: Active Active Active
4232025-10-22T21:50:12.757ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4242025-10-22T21:50:12.757Z 1002 Acked Read 50 Done Done Done false
4252025-10-22T21:50:12.757Z STATES DS:0 DS:1 DS:2 TOTAL
4262025-10-22T21:50:12.757Z Sent 0 0 0 0
4272025-10-22T21:50:12.757Z Done 1 1 1 3
4282025-10-22T21:50:12.757Z Skipped 0 0 0 0
4292025-10-22T21:50:12.757Z Error 0 0 0 0
4302025-10-22T21:50:12.757ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4312025-10-22T21:50:12.757ZDownstairs last five completed: 1001 1000
4322025-10-22T21:50:12.757ZUpstairs last five completed: 1002 1001 1000
4332025-10-22T21:50:12.757ZCLIENT: Up:0 ds:2 act:6
4342025-10-22T21:50:16.756Z----------------------------------------------------------------
4352025-10-22T21:50:16.756Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4362025-10-22T21:50:16.756ZDownstairs last five completed: 1003 1002 1001 1000
4372025-10-22T21:50:16.756ZUpstairs last five completed: 1003 1002 1001 1000
4382025-10-22T21:50:16.756Z----------------------------------------------------------------
4392025-10-22T21:50:16.756Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4402025-10-22T21:50:16.756ZDownstairs last five completed: 1003 1002 1001 1000
4412025-10-22T21:50:16.756ZUpstairs last five completed: 1003 1002 1001 1000
4422025-10-22T21:50:16.760ZCLIENT: Up:0 ds:0 act:6
4432025-10-22T21:50:16.760ZCLIENT: All crucible jobs finished, exiting program
4442025-10-22T21:50:16.763ZRepair part 1 passed
4452025-10-22T21:50:16.763Z
4462025-10-22T21:50:16.763ZKill the current downstairs
4472025-10-22T21:50:16.938ZDownstairs 2 stopped
4482025-10-22T21:50:16.938Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
4492025-10-22T21:50:16.941ZNow put back the original so we have a mismatch
4502025-10-22T21:50:16.941Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
4512025-10-22T21:50:16.944ZRestart downstairs with old directory
4522025-10-22T21:50:17.001Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4532025-10-22T21:50:17.021ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4542025-10-22T21:50:17.025Z 0 000-009 32 32 31 4005 4005 4004 F F F <---
4552025-10-22T21:50:17.025Z 1 010-019 32 32 31 4005 4005 4004 F F F <---
4562025-10-22T21:50:17.025Z 2 020-029 32 32 31 4005 4005 4004 F F F <---
4572025-10-22T21:50:17.025Z 3 030-039 32 32 31 4005 4005 4004 F F F <---
4582025-10-22T21:50:17.025Z 4 040-049 32 32 31 4005 4005 4004 F F F <---
4592025-10-22T21:50:17.025ZMax gen: 32, Max flush: 4005
4602025-10-22T21:50:17.025ZError: Difference in extent metadata found!
4612025-10-22T21:50:17.028Zdump test found error as expected
4622025-10-22T21:50:17.028Z
4632025-10-22T21:50:17.028Z
4642025-10-22T21:50:17.028Z/var/tmp/bins/crutest verify --range -g 33 -q --key Mh5+cDXnjTwiXFBStt7r54PzNsgmw6A08uZqaayXxAI= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
4652025-10-22T21:50:17.087Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.852657102Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"task":"crutest"}
4662025-10-22T21:50:17.091Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.853328201Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"task":"crutest"}
4672025-10-22T21:50:17.091Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.85520534Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"task":"crutest"}
4682025-10-22T21:50:17.091Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.855435122Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260}
4692025-10-22T21:50:17.091Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9063dde9565406ad9efc6e457bbffa920b2c2351\",\n git_commit_timestamp: \"2025-10-22T21:23:18.000000000Z\",\n git_branch: \"mkeeter/check-hashes\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.85550726Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260}
4702025-10-22T21:50:17.091Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.855542554Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260}
4712025-10-22T21:50:17.091Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.855594136Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260}
4722025-10-22T21:50:17.091Z{"msg":"Crucible f7a09b00-8b95-45d3-b4eb-abb03e802af4 has session id: d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.856117389Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
4732025-10-22T21:50:17.091Z{"msg":"Upstairs opts: Upstairs UUID: f7a09b00-8b95-45d3-b4eb-abb03e802af4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.856176009Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
4742025-10-22T21:50:17.091Z{"msg":"Crucible stats registered with UUID: f7a09b00-8b95-45d3-b4eb-abb03e802af4","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.85623327Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
4752025-10-22T21:50:17.091Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-10-22T21:50:15.856271203Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
4762025-10-22T21:50:17.091Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.856357019Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"io task","client":"1","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
4772025-10-22T21:50:17.091Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.856408881Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"io task","client":"0","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
4782025-10-22T21:50:17.091Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.856448614Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
4792025-10-22T21:50:17.095Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.858367206Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"task":"crutest"}
4802025-10-22T21:50:17.095Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.858458801Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260}
4812025-10-22T21:50:17.095Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9063dde9565406ad9efc6e457bbffa920b2c2351\",\n git_commit_timestamp: \"2025-10-22T21:23:18.000000000Z\",\n git_branch: \"mkeeter/check-hashes\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.858528939Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260}
4822025-10-22T21:50:17.095Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.858559025Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260}
4832025-10-22T21:50:17.096Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.858593859Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260}
4842025-10-22T21:50:17.096Z{"msg":"Crucible f7a09b00-8b95-45d3-b4eb-abb03e802af4 has session id: 4090efd2-f666-4c8d-905f-8926c8db4acc","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.8589463Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
4852025-10-22T21:50:17.096Z{"msg":"Upstairs opts: Upstairs UUID: f7a09b00-8b95-45d3-b4eb-abb03e802af4, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.859019608Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
4862025-10-22T21:50:17.096Z{"msg":"Crucible stats registered with UUID: f7a09b00-8b95-45d3-b4eb-abb03e802af4","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.859056322Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
4872025-10-22T21:50:17.096Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-10-22T21:50:15.859078568Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
4882025-10-22T21:50:17.096Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.859099185Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"io task","client":"0","":"downstairs","session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
4892025-10-22T21:50:17.096Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.859133629Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"io task","client":"1","":"downstairs","session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
4902025-10-22T21:50:17.096Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.859163154Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
4912025-10-22T21:50:17.096Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.859880735Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"task":"crutest"}
4922025-10-22T21:50:17.096Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.85991511Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260}
4932025-10-22T21:50:17.096Z{"msg":"f7a09b00-8b95-45d3-b4eb-abb03e802af4 active request set","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.860016323Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
4942025-10-22T21:50:17.096Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.860056686Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"client":"0","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
4952025-10-22T21:50:17.096Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.860075763Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"client":"1","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
4962025-10-22T21:50:17.096Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.860105718Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"client":"2","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
4972025-10-22T21:50:17.096Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.860127465Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
4982025-10-22T21:50:17.096Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.860163369Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"io task","client":"0","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
4992025-10-22T21:50:17.096Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.860191554Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"io task","client":"1","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5002025-10-22T21:50:17.096Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-10-22T21:50:15.860306085Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5012025-10-22T21:50:17.096Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-10-22T21:50:15.860343769Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5022025-10-22T21:50:17.096Z{"msg":"ds_connection connected from Ok(127.0.0.1:44429)","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.860362966Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"io task","client":"1","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5032025-10-22T21:50:17.096Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-10-22T21:50:15.860402119Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5042025-10-22T21:50:17.096Z{"msg":"ds_connection connected from Ok(127.0.0.1:52324)","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.860427125Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"io task","client":"0","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5052025-10-22T21:50:17.096Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.860452711Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5062025-10-22T21:50:17.096Z{"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.860489495Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5072025-10-22T21:50:17.096Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.86154122Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"client":"1","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5082025-10-22T21:50:17.096Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.861589103Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"client":"0","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5092025-10-22T21:50:17.096Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.861956902Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5102025-10-22T21:50:17.096Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:15.862005594Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5112025-10-22T21:50:27.096Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.861270316Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5122025-10-22T21:50:27.102Z{"msg":"ds_connection connected from Ok(127.0.0.1:46646)","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.861448116Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5132025-10-22T21:50:27.102Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.862860162Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"client":"2","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5142025-10-22T21:50:27.102Z{"msg":"[0]R flush_numbers: [4005, 4005, 4005, 4005, 4005]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.863260596Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"downstairs","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5152025-10-22T21:50:27.102Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.86329469Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"downstairs","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5162025-10-22T21:50:27.102Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.863314157Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"downstairs","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5172025-10-22T21:50:27.102Z{"msg":"[1]R flush_numbers: [4005, 4005, 4005, 4005, 4005]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.863339773Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"downstairs","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5182025-10-22T21:50:27.102Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.86335723Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"downstairs","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5192025-10-22T21:50:27.102Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.863381396Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"downstairs","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5202025-10-22T21:50:27.102Z{"msg":"[2]R flush_numbers: [4004, 4004, 4004, 4004, 4004]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.863399163Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"downstairs","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5212025-10-22T21:50:27.102Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.863422909Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"downstairs","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5222025-10-22T21:50:27.102Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.863441716Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"downstairs","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5232025-10-22T21:50:27.102Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.863465402Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"downstairs","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5242025-10-22T21:50:27.102Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.863483909Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"downstairs","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5252025-10-22T21:50:27.102Z{"msg":"Next flush: 4006","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.863506475Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"downstairs","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5262025-10-22T21:50:27.102Z{"msg":"extent 0 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { gen: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { gen: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { gen: 31, flush: 4004, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.863532231Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5272025-10-22T21:50:27.102Z{"msg":"extent:0 ClientMap(ClientData([Some(ExtentMetadata { gen: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { gen: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { gen: 31, flush: 4004, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.863558367Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5282025-10-22T21:50:27.102Z{"msg":"found dest for source 0 for extent at index 0 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.863583442Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5292025-10-22T21:50:27.102Z{"msg":"extent 1 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { gen: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { gen: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { gen: 31, flush: 4004, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.863609368Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5302025-10-22T21:50:27.102Z{"msg":"extent:1 ClientMap(ClientData([Some(ExtentMetadata { gen: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { gen: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { gen: 31, flush: 4004, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.863634694Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5312025-10-22T21:50:27.102Z{"msg":"found dest for source 0 for extent at index 1 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.86365989Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5322025-10-22T21:50:27.102Z{"msg":"extent 2 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { gen: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { gen: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { gen: 31, flush: 4004, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.863685476Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5332025-10-22T21:50:27.102Z{"msg":"extent:2 ClientMap(ClientData([Some(ExtentMetadata { gen: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { gen: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { gen: 31, flush: 4004, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.863710352Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5342025-10-22T21:50:27.102Z{"msg":"found dest for source 0 for extent at index 2 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.863735417Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5352025-10-22T21:50:27.102Z{"msg":"extent 3 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { gen: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { gen: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { gen: 31, flush: 4004, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.863761303Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5362025-10-22T21:50:27.103Z{"msg":"extent:3 ClientMap(ClientData([Some(ExtentMetadata { gen: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { gen: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { gen: 31, flush: 4004, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.863792138Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5372025-10-22T21:50:27.103Z{"msg":"found dest for source 0 for extent at index 3 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.863829042Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5382025-10-22T21:50:27.103Z{"msg":"extent 4 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { gen: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { gen: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { gen: 31, flush: 4004, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.863871245Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5392025-10-22T21:50:27.103Z{"msg":"extent:4 ClientMap(ClientData([Some(ExtentMetadata { gen: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { gen: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { gen: 31, flush: 4004, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.863906219Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5402025-10-22T21:50:27.103Z{"msg":"found dest for source 0 for extent at index 4 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.863926156Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5412025-10-22T21:50:27.103Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.86396281Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"client":"0","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5422025-10-22T21:50:27.103Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.86401934Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"client":"1","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5432025-10-22T21:50:27.103Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.864071502Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"client":"2","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5442025-10-22T21:50:27.103Z{"msg":"Full repair list: {ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.864129402Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"downstairs","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5452025-10-22T21:50:27.103Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 4006, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 4006, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 4006, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 4006, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 4006, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.864347446Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"downstairs","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5462025-10-22T21:50:27.103Z{"msg":"starting reconciliation 1d43a66c-e90d-4a55-98e1-fb04e413b7a8: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.864374642Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"downstairs","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5472025-10-22T21:50:27.103Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.864391949Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"client":"2","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5482025-10-22T21:50:27.109Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.873954455Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"client":"2","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5492025-10-22T21:50:27.114Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.877922088Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"client":"2","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5502025-10-22T21:50:27.117Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.882017689Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"client":"2","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5512025-10-22T21:50:27.121Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.885967235Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"client":"2","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5522025-10-22T21:50:27.124Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.889483513Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"downstairs","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5532025-10-22T21:50:27.124Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.889527896Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"client":"0","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5542025-10-22T21:50:27.128Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.889570189Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"client":"1","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5552025-10-22T21:50:27.128Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.889612631Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"client":"2","":"downstairs","session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5562025-10-22T21:50:27.128Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.889643896Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5572025-10-22T21:50:27.128Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.889666973Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5582025-10-22T21:50:27.128Z{"msg":"f7a09b00-8b95-45d3-b4eb-abb03e802af4 is now active with session: d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.889693818Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"session_id":"d6e0855d-e1f9-4b5d-99e2-82949ebb7cd7"}
5592025-10-22T21:50:27.128Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.889729732Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260}
5602025-10-22T21:50:27.128Z{"msg":"Activated sub_volume f7a09b00-8b95-45d3-b4eb-abb03e802af4","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.889753748Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260}
5612025-10-22T21:50:27.128Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.889782623Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260}
5622025-10-22T21:50:27.128Z{"msg":"f7a09b00-8b95-45d3-b4eb-abb03e802af4 active request set","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.889806569Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5632025-10-22T21:50:27.128Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.889832545Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"client":"0","":"downstairs","session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5642025-10-22T21:50:27.128Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.88986072Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"client":"1","":"downstairs","session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5652025-10-22T21:50:27.128Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.889889176Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"client":"2","":"downstairs","session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5662025-10-22T21:50:27.128Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.889916851Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5672025-10-22T21:50:27.128Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.889946286Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"io task","client":"0","":"downstairs","session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5682025-10-22T21:50:27.128Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.889978031Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"io task","client":"1","":"downstairs","session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5692025-10-22T21:50:27.128Z{"msg":"ds_connection connected from Ok(127.0.0.1:37834)","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.890013945Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5702025-10-22T21:50:27.128Z{"msg":"ds_connection connected from Ok(127.0.0.1:48486)","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.890049049Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"io task","client":"0","":"downstairs","session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5712025-10-22T21:50:27.128Z{"msg":"ds_connection connected from Ok(127.0.0.1:64816)","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.890085613Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"io task","client":"1","":"downstairs","session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5722025-10-22T21:50:27.129Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.890810003Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"client":"1","":"downstairs","session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5732025-10-22T21:50:27.129Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.890854986Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"client":"2","":"downstairs","session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5742025-10-22T21:50:27.129Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.8908898Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"client":"0","":"downstairs","session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5752025-10-22T21:50:27.129Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.891161615Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5762025-10-22T21:50:27.129Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.891268837Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5772025-10-22T21:50:27.129Z{"msg":"[0]R flush_numbers: [5320, 5320, 5320, 5320, 5320]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.891318629Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"downstairs","":"downstairs","session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5782025-10-22T21:50:27.129Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.891361942Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"downstairs","":"downstairs","session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5792025-10-22T21:50:27.129Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.891406254Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"downstairs","":"downstairs","session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5802025-10-22T21:50:27.129Z{"msg":"[1]R flush_numbers: [5320, 5320, 5320, 5320, 5320]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
5812025-10-22T21:50:27.129Z2025-10-22T21:50:25.891448517Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"downstairs","":"downstairs","session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5822025-10-22T21:50:27.129Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.891495459Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"downstairs","":"downstairs","session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5832025-10-22T21:50:27.129Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.891537073Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"downstairs","":"downstairs","session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5842025-10-22T21:50:27.129Z{"msg":"[2]R flush_numbers: [5320, 5320, 5320, 5320, 5320]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.891579186Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"downstairs","":"downstairs","session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5852025-10-22T21:50:27.129Z{"msg":"[2]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.891626028Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"downstairs","":"downstairs","session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5862025-10-22T21:50:27.129Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.89167497Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"downstairs","":"downstairs","session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5872025-10-22T21:50:27.129Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.891719213Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"downstairs","":"downstairs","session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5882025-10-22T21:50:27.129Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.891762685Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"downstairs","":"downstairs","session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5892025-10-22T21:50:27.129Z{"msg":"Next flush: 5321","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.891802779Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"downstairs","":"downstairs","session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5902025-10-22T21:50:27.129ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
5912025-10-22T21:50:27.129Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.891849211Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"":"downstairs","":"downstairs","session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5922025-10-22T21:50:27.129Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.891874927Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5932025-10-22T21:50:27.129Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.891898823Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"client":"0","":"downstairs","session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5942025-10-22T21:50:27.129Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.89191594Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"client":"1","":"downstairs","session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5952025-10-22T21:50:27.129Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.891939116Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"client":"2","":"downstairs","session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5962025-10-22T21:50:27.129Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.891956023Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5972025-10-22T21:50:27.129Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.89197881Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5982025-10-22T21:50:27.129Z{"msg":"f7a09b00-8b95-45d3-b4eb-abb03e802af4 is now active with session: 4090efd2-f666-4c8d-905f-8926c8db4acc","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.892006785Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
5992025-10-22T21:50:27.129ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
6002025-10-22T21:50:27.129Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.89203583Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260,"session_id":"4090efd2-f666-4c8d-905f-8926c8db4acc"}
6012025-10-22T21:50:27.129Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.892058706Z","hostname":"Read and Verify all blocks (0..100 range:true)
6022025-10-22T21:50:27.129Zw-01K86XZW6S6HABWSWXH7785A6V","pid":1260}
6032025-10-22T21:50:27.129Z{"msg":"Activated sub_volume f7a09b00-8b95-45d3-b4eb-abb03e802af4","v":0,"name":"crucible","level":30,"time":"2025-10-22T21:50:25.892075023Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260}
6042025-10-22T21:50:27.209ZWrote out file "/var/tmp/test_up-build/verify_file"
6052025-10-22T21:50:27.210ZVerify test completed
6062025-10-22T21:50:27.213ZWrote out file "/var/tmp/test_up-build/verify_file"
6072025-10-22T21:50:27.213ZCLIENT: Tests done. All submitted work has been ACK'd
6082025-10-22T21:50:27.213Z----------------------------------------------------------------
6092025-10-22T21:50:27.213Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6102025-10-22T21:50:27.213ZStates: Active Active Active
6112025-10-22T21:50:27.213ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6122025-10-22T21:50:27.213Z 1000 Acked Read 50 Done Done Done false
6132025-10-22T21:50:27.213Z STATES DS:0 DS:1 DS:2 TOTAL
6142025-10-22T21:50:27.213Z Sent 0 0 0 0
6152025-10-22T21:50:27.213Z Done 1 1 1 3
6162025-10-22T21:50:27.213Z Skipped 0 0 0 0
6172025-10-22T21:50:27.213Z Error 0 0 0 0
6182025-10-22T21:50:27.213ZLast Flush: None None None
6192025-10-22T21:50:27.213ZDownstairs last five completed:
6202025-10-22T21:50:27.213ZUpstairs last five completed: 1000
6212025-10-22T21:50:27.213Z----------------------------------------------------------------
6222025-10-22T21:50:27.213Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6232025-10-22T21:50:27.214ZStates: Active Active Active
6242025-10-22T21:50:27.214ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6252025-10-22T21:50:27.214Z 1000 Acked Read 50 Sent Done Sent false
6262025-10-22T21:50:27.214Z STATES DS:0 DS:1 DS:2 TOTAL
6272025-10-22T21:50:27.214Z Sent 1 0 1 2
6282025-10-22T21:50:27.214Z Done 0 1 0 1
6292025-10-22T21:50:27.214Z Skipped 0 0 0 0
6302025-10-22T21:50:27.214Z Error 0 0 0 0
6312025-10-22T21:50:27.214ZLast Flush: None None None
6322025-10-22T21:50:27.214ZDownstairs last five completed:
6332025-10-22T21:50:27.214ZUpstairs last five completed: 1000
6342025-10-22T21:50:27.214ZCLIENT: Up:0 ds:2 act:6
6352025-10-22T21:50:31.212Z----------------------------------------------------------------
6362025-10-22T21:50:31.212Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6372025-10-22T21:50:31.212ZDownstairs last five completed: 1001 1000
6382025-10-22T21:50:31.212ZUpstairs last five completed: 1001 1000
6392025-10-22T21:50:31.212Z----------------------------------------------------------------
6402025-10-22T21:50:31.212Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6412025-10-22T21:50:31.212ZDownstairs last five completed: 1001 1000
6422025-10-22T21:50:31.212ZUpstairs last five completed: 1001 1000
6432025-10-22T21:50:31.216ZCLIENT: Up:0 ds:0 act:6
6442025-10-22T21:50:31.216ZCLIENT: All crucible jobs finished, exiting program
6452025-10-22T21:50:31.216Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-10-22T21:50:29.976491248Z","hostname":"w-01K86XZW6S6HABWSWXH7785A6V","pid":1260}
6462025-10-22T21:50:31.219ZRepair part 2 passed
6472025-10-22T21:50:31.222Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
6482025-10-22T21:50:31.240ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6492025-10-22T21:50:31.244Z 0 000-009 32 32 32 4005 4005 4005 F F F
6502025-10-22T21:50:31.244Z 1 010-019 32 32 32 4005 4005 4005 F F F
6512025-10-22T21:50:31.244Z 2 020-029 32 32 32 4005 4005 4005 F F F
6522025-10-22T21:50:31.244Z 3 030-039 32 32 32 4005 4005 4005 F F F
6532025-10-22T21:50:31.244Z 4 040-049 32 32 32 4005 4005 4005 F F F
6542025-10-22T21:50:31.244ZMax gen: 32, Max flush: 4005
6552025-10-22T21:50:31.247Zdump test passed
6562025-10-22T21:50:31.247Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
6572025-10-22T21:50:31.265Z Extent 1
6582025-10-22T21:50:31.268ZGEN 32 32 32
6592025-10-22T21:50:31.269ZFLUSH_ID 4005 4005 4005
6602025-10-22T21:50:31.269ZDIRTY
6612025-10-22T21:50:31.269Z
6622025-10-22T21:50:31.269ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
6632025-10-22T21:50:31.272Z010 A A A A A A
6642025-10-22T21:50:31.275Z011 A A A A A A
6652025-10-22T21:50:31.278Z012 A A A A A A
6662025-10-22T21:50:31.282Z013 A A A A A A
6672025-10-22T21:50:31.285Z014 A A A A A A
6682025-10-22T21:50:31.288Z015 A A A A A A
6692025-10-22T21:50:31.291Z016 A A A A A A
6702025-10-22T21:50:31.295Z017 A A A A A A
6712025-10-22T21:50:31.298Z018 A A A A A A
6722025-10-22T21:50:31.301Z019 A A A A A A
6732025-10-22T21:50:31.305Zdump extent test passed
6742025-10-22T21:50:31.308Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
6752025-10-22T21:50:31.325ZExtent 2 Block in extent 0 Actual block 20
6762025-10-22T21:50:31.328Z
6772025-10-22T21:50:31.332Z DATA SHA256 VER
6782025-10-22T21:50:31.332Z------ ---------------------------------------------------------------- ---
6792025-10-22T21:50:31.332Z 0 9a672dcec3ad79d8bb41904777a22f2910318c50c60b9693886943450497cbb4 A
6802025-10-22T21:50:31.333Z 1 9a672dcec3ad79d8bb41904777a22f2910318c50c60b9693886943450497cbb4 A
6812025-10-22T21:50:31.333Z 2 9a672dcec3ad79d8bb41904777a22f2910318c50c60b9693886943450497cbb4 A
6822025-10-22T21:50:31.333Z
6832025-10-22T21:50:31.333ZNONCES 0 1 2 DIFF
6842025-10-22T21:50:31.333Z------ ------------------------ ------------------------ ------------------------ -----
6852025-10-22T21:50:31.333Z 0 37562c5a303387c6490efa43 37562c5a303387c6490efa43 37562c5a303387c6490efa43
6862025-10-22T21:50:31.333Z
6872025-10-22T21:50:31.333Z TAGS 0 1 2 DIFF
6882025-10-22T21:50:31.333Z------ -------------------------------- -------------------------------- -------------------------------- -----
6892025-10-22T21:50:31.333Z 0 cc89bd628045dde2db3c89907559c975 cc89bd628045dde2db3c89907559c975 cc89bd628045dde2db3c89907559c975
6902025-10-22T21:50:31.333Z
6912025-10-22T21:50:31.333ZHASHES 0 1 2 DIFF
6922025-10-22T21:50:31.333Z------ ---------------- ---------------- ---------------- -----
6932025-10-22T21:50:31.333Z
6942025-10-22T21:50:31.336Zdump block test passed
6952025-10-22T21:50:31.336ZInitial upstairs tests have completed, stopping all downstairs
6962025-10-22T21:50:36.403ZCreating 7 larger downstairs regions
6972025-10-22T21:50:39.146ZStarting 7 downstairs
6982025-10-22T21:50:44.153Zdsc restarted at PID: 1276
6992025-10-22T21:50:44.556ZNow do the replace-reconcile test
7002025-10-22T21:50:44.614ZUsing 8870 for the replacement port
7012025-10-22T21:54:23.881ZCompleted test: replace-reconcile
7022025-10-22T21:54:23.884ZNow do the replace-before-active test
7032025-10-22T22:00:30.059ZCompleted test: replace-before-active
7042025-10-22T22:00:30.062ZAll tests have completed, stopping all downstairs
7052025-10-22T22:00:30.121Z
7062025-10-22T22:00:30.121ZAll Tests have passed
7072025-10-22T22:00:30.125Z16:13 Test duration
7082025-10-22T22:00:30.125Z
7092025-10-22T22:00:30.125Zreal 16:13.417891776
7102025-10-22T22:00:30.125Zuser 40:33.183257978
7112025-10-22T22:00:30.125Zsys 32.086040695
7122025-10-22T22:00:30.125Ztrap 1.371544928
7132025-10-22T22:00:30.125Ztflt 0.058551975
7142025-10-22T22:00:30.125Zdflt 0.076740923
7152025-10-22T22:00:30.127Zkflt 0.001381085
7162025-10-22T22:00:30.127Zlock 24:07:22.174473545
7172025-10-22T22:00:30.127Zslp 1:14:11.875645851
7182025-10-22T22:00:30.127Zlat 5:12.848997694
7192025-10-22T22:00:30.127Zstop 0.061445088
7202025-10-22T22:00:30.127Z+ echo 'test-up-2region-encrypted ends'
7212025-10-22T22:00:30.127Ztest-up-2region-encrypted ends
7222025-10-22T22:00:35.128Zprocess exited: duration 983504 ms, exit code 0
7232025-10-22T22:00:35.128Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
7242025-10-22T22:01:35.167Zexec warning: : stdout descriptor may be held open by a background process; giving up!
7252025-10-22T22:01:35.167Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
7262025-10-22T22:01:35.185Zfound 17 output files
7272025-10-22T22:01:35.185Zuploading: /tmp/test_up-build/dsc-out.txt (54400 bytes)
7282025-10-22T22:01:36.205Zuploaded: /tmp/test_up-build/dsc-out.txt
7292025-10-22T22:01:36.208Zuploading: /tmp/test_up-build/test_up_out.txt (19947494 bytes)
7302025-10-22T22:01:37.405Zuploaded: /tmp/test_up-build/test_up_out.txt
7312025-10-22T22:01:37.409Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (550846 bytes)
7322025-10-22T22:01:38.432Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
7332025-10-22T22:01:38.435Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (548271 bytes)
7342025-10-22T22:01:39.456Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
7352025-10-22T22:01:39.459Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (979481 bytes)
7362025-10-22T22:01:40.492Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
7372025-10-22T22:01:40.495Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (543049 bytes)
7382025-10-22T22:01:41.519Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
7392025-10-22T22:01:41.524Zuploading: /tmp/test_up-build/dsc/downstairs-8850.txt (2496 bytes)
7402025-10-22T22:01:42.534Zuploaded: /tmp/test_up-build/dsc/downstairs-8850.txt
7412025-10-22T22:01:42.538Zuploading: /tmp/test_up-build/dsc/downstairs-8860.txt (39006 bytes)
7422025-10-22T22:01:43.550Zuploaded: /tmp/test_up-build/dsc/downstairs-8860.txt
7432025-10-22T22:01:43.553Zuploading: /tmp/test_up-build/dsc/downstairs-8870.txt (3620406 bytes)
7442025-10-22T22:01:44.599Zuploaded: /tmp/test_up-build/dsc/downstairs-8870.txt
7452025-10-22T22:01:44.599Zuploading: /tmp/debug/df.txt (1270 bytes)
7462025-10-22T22:01:45.610Zuploaded: /tmp/debug/df.txt
7472025-10-22T22:01:45.613Zuploading: /tmp/debug/dtrace.txt (1391996 bytes)
7482025-10-22T22:01:46.644Zuploaded: /tmp/debug/dtrace.txt
7492025-10-22T22:01:46.647Zuploading: /tmp/debug/iostat.txt (311929 bytes)
7502025-10-22T22:01:46.659Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 311929 -> 315229
7512025-10-22T22:01:47.667Zuploaded: /tmp/debug/iostat.txt
7522025-10-22T22:01:47.667Zuploading: /tmp/debug/mpstat.txt (757441 bytes)
7532025-10-22T22:01:47.689Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 757441 -> 766189
7542025-10-22T22:01:48.697Zuploaded: /tmp/debug/mpstat.txt
7552025-10-22T22:01:48.697Zuploading: /tmp/debug/paging.txt (135917 bytes)
7562025-10-22T22:01:48.712Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 135917 -> 137728
7572025-10-22T22:01:49.720Zuploaded: /tmp/debug/paging.txt
7582025-10-22T22:01:49.720Zuploading: /tmp/debug/prstat.txt (1500644 bytes)
7592025-10-22T22:01:49.748Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 1500644 -> 1518700
7602025-10-22T22:01:50.758Zuploaded: /tmp/debug/prstat.txt
7612025-10-22T22:01:50.761Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
7622025-10-22T22:01:51.770Zuploaded: /tmp/debug/psrinfo.txt
7632025-10-22T22:01:51.774Zuploading: /tmp/debug/upstairs-info.txt (235739 bytes)
7642025-10-22T22:01:52.792Zuploaded: /tmp/debug/upstairs-info.txt