01KYKFDAQV4NY0M62MXF6KHD8V: test-up-2region-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01KYKFDSSDJNKVK9KH710AXWRT

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12026-07-28T04:37:09.515Zjob dependencies complete; ready to run (waiting for 13 m 22 s)
22026-07-28T04:38:09.914Zjob assigned to worker 01KYKG6JQGQ8Z9NT39M0RRXVEB [factory aws, i-0d4864cf50f0bdcfc] (queued for 1 m 0 s)
32026-07-28T04:38:17.182Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42026-07-28T04:38:19.133Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52026-07-28T04:38:19.138Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62026-07-28T04:38:21.101Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72026-07-28T04:38:21.110Zdownloading input: /input/build/work/bins/crutest.gz
82026-07-28T04:38:22.902Zdownloaded input: /input/build/work/bins/crutest.gz
92026-07-28T04:38:22.906Zdownloading input: /input/build/work/bins/dsc.gz
102026-07-28T04:38:23.903Zdownloaded input: /input/build/work/bins/dsc.gz
112026-07-28T04:38:23.912Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122026-07-28T04:38:23.939Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132026-07-28T04:38:23.944Zdownloading input: /input/build/work/scripts/test_ds.sh
142026-07-28T04:38:23.971Zdownloaded input: /input/build/work/scripts/test_ds.sh
152026-07-28T04:38:23.975Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162026-07-28T04:38:24.000Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172026-07-28T04:38:24.007Zdownloading input: /input/build/work/scripts/test_repair.sh
182026-07-28T04:38:24.030Zdownloaded input: /input/build/work/scripts/test_repair.sh
192026-07-28T04:38:24.030Zdownloading input: /input/build/work/scripts/test_replay.sh
202026-07-28T04:38:24.063Zdownloaded input: /input/build/work/scripts/test_replay.sh
212026-07-28T04:38:24.070Zdownloading input: /input/build/work/scripts/test_up.sh
222026-07-28T04:38:24.094Zdownloaded input: /input/build/work/scripts/test_up.sh
232026-07-28T04:38:24.094Zdownloading input: /input/build/work/scripts/upstairs_info.d
242026-07-28T04:38:24.127Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252026-07-28T04:38:24.132Zdownloading input: /input/build/tmp/cargo-test-out.log
262026-07-28T04:38:24.851Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272026-07-28T04:38:24.851Zstarting task 0: "setup"
282026-07-28T04:38:24.858Z++ uname -s
292026-07-28T04:38:24.858Z+ kern=SunOS
302026-07-28T04:38:24.858Z+ build_user=build
312026-07-28T04:38:24.858Z+ build_uid=12345
322026-07-28T04:38:24.858Z+ work_dir=/work
332026-07-28T04:38:24.858Z+ input_dir=/input
342026-07-28T04:38:24.858Z+ [[ 0 == 12345 ]]
352026-07-28T04:38:24.858Z+ case "$kern" in
362026-07-28T04:38:24.858Z+ groupadd -g 12345 build
372026-07-28T04:38:24.865Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382026-07-28T04:38:26.867Z+ zfs create -o mountpoint=/work rpool/work
392026-07-28T04:38:27.234Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402026-07-28T04:38:27.245Z+ home_fs=zfs
412026-07-28T04:38:27.245Z+ [[ zfs == autofs ]]
422026-07-28T04:38:27.245Z+ mkdir -p /home/build
432026-07-28T04:38:27.245Z+ chown build:build /home/build /work
442026-07-28T04:38:29.351Z+ chmod 0700 /home/build /work
452026-07-28T04:38:29.381Zprocess exited: duration 4385 ms, exit code 0
 
462026-07-28T04:38:29.418Zstarting task 1: "authentication"
472026-07-28T04:38:29.452Zprocess exited: duration 30 ms, exit code 0
 
482026-07-28T04:38:29.498Zstarting task 2: "build"
492026-07-28T04:38:29.596Z+ banner cores
502026-07-28T04:38:29.596Z
512026-07-28T04:38:29.596Z #### #### ##### ###### ####
522026-07-28T04:38:29.596Z # # # # # # # #
532026-07-28T04:38:29.596Z # # # # # ##### ####
542026-07-28T04:38:29.596Z # # # ##### # #
552026-07-28T04:38:29.596Z # # # # # # # # #
562026-07-28T04:38:29.597Z #### #### # # ###### ####
572026-07-28T04:38:29.597Z
582026-07-28T04:38:29.597Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
592026-07-28T04:38:29.597Z+ banner unpack
602026-07-28T04:38:29.597Z
612026-07-28T04:38:29.597Z # # # # ##### ## #### # #
622026-07-28T04:38:29.597Z # # ## # # # # # # # # #
632026-07-28T04:38:29.597Z # # # # # # # # # # ####
642026-07-28T04:38:29.597Z # # # # # ##### ###### # # #
652026-07-28T04:38:29.597Z # # # ## # # # # # # #
662026-07-28T04:38:29.597Z #### # # # # # #### # #
672026-07-28T04:38:29.597Z
682026-07-28T04:38:29.597Z+ mkdir -p /var/tmp/bins
692026-07-28T04:38:29.597Z+ for t in "$input/bins/"*.gz
702026-07-28T04:38:29.597Z++ basename /input/build/work/bins/crucible-downstairs.gz
712026-07-28T04:38:29.597Z+ b=crucible-downstairs.gz
722026-07-28T04:38:29.597Z+ b=crucible-downstairs
732026-07-28T04:38:29.597Z+ gunzip
742026-07-28T04:38:32.540Z+ chmod +x /var/tmp/bins/crucible-downstairs
752026-07-28T04:38:32.545Z+ for t in "$input/bins/"*.gz
762026-07-28T04:38:32.545Z++ basename /input/build/work/bins/crucible-hammer.gz
772026-07-28T04:38:32.545Z+ b=crucible-hammer.gz
782026-07-28T04:38:32.546Z+ b=crucible-hammer
792026-07-28T04:38:32.546Z+ gunzip
802026-07-28T04:38:35.472Z+ chmod +x /var/tmp/bins/crucible-hammer
812026-07-28T04:38:35.485Z+ for t in "$input/bins/"*.gz
822026-07-28T04:38:35.485Z++ basename /input/build/work/bins/crutest.gz
832026-07-28T04:38:35.485Z+ b=crutest.gz
842026-07-28T04:38:35.485Z+ b=crutest
852026-07-28T04:38:35.485Z+ gunzip
862026-07-28T04:38:38.506Z+ chmod +x /var/tmp/bins/crutest
872026-07-28T04:38:38.546Z+ for t in "$input/bins/"*.gz
882026-07-28T04:38:38.546Z++ basename /input/build/work/bins/dsc.gz
892026-07-28T04:38:38.546Z+ b=dsc.gz
902026-07-28T04:38:38.547Z+ b=dsc
912026-07-28T04:38:38.547Z+ gunzip
922026-07-28T04:38:43.824Z+ chmod +x /var/tmp/bins/dsc
932026-07-28T04:38:43.875Z+ export BINDIR=/var/tmp/bins
942026-07-28T04:38:43.875Z+ BINDIR=/var/tmp/bins
952026-07-28T04:38:43.875Z+ jobpid=1091
962026-07-28T04:38:43.875Z+ echo 'Setup debug logging'
972026-07-28T04:38:43.875ZSetup debug logging
982026-07-28T04:38:43.875Z+ mkdir /tmp/debug
992026-07-28T04:38:43.876Z+ sleep 3600
1002026-07-28T04:38:43.876Z+ psrinfo -v
1012026-07-28T04:38:43.876Z+ df -h
1022026-07-28T04:38:43.876Z+ prstat -d d -mLc 1
1032026-07-28T04:38:43.876Z+ iostat -T d -xn 1
1042026-07-28T04:38:43.876Z+ mpstat -T d 1
1052026-07-28T04:38:43.876Z+ vmstat -T d -p 1
1062026-07-28T04:38:43.876Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1072026-07-28T04:38:43.876Z+ banner test_up_2r_encrypted
1082026-07-28T04:38:43.876Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1092026-07-28T04:38:43.876Z #####
1102026-07-28T04:38:43.876Z ##### ###### #### ##### # # ##### # # #####
1112026-07-28T04:38:43.876Z # # # # # # # # # # #
1122026-07-28T04:38:43.876Z # ##### #### # # # # # ##### # #
1132026-07-28T04:38:43.876Z # # # # # # ##### # #####
1142026-07-28T04:38:43.876Z # # # # # # # # # # #
1152026-07-28T04:38:43.876Z # ###### #### # ####### #### # ####### ####### # #
1162026-07-28T04:38:43.876Z
1172026-07-28T04:38:43.876Z+ ptime -m bash /input/build/work/scripts/test_up.sh -r 2 -N encrypted
1182026-07-28T04:38:43.876Z/input/build/work
1192026-07-28T04:38:43.876ZUsing 2 region sets
1202026-07-28T04:38:43.876ZTurn off color for downstairs dump
1212026-07-28T04:38:43.876ZUpstairs using key: 3nhxQym94pYVbi5RDOWrNoPrbwhDkUuuDP5Cs9MKUFc=
1222026-07-28T04:38:43.876Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1232026-07-28T04:38:43.876ZCreating 6 downstairs regions
1242026-07-28T04:38:43.876ZStarting 6 downstairs
1252026-07-28T04:38:43.876Zdsc started at PID: 1139
1262026-07-28T04:38:45.158ZDisable automatic restart on all downstairs
1272026-07-28T04:38:45.170Z
1282026-07-28T04:38:45.170ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1292026-07-28T04:38:45.173ZRunning test: span
1302026-07-28T04:38:49.278ZCompleted test: span
1312026-07-28T04:38:54.284ZRunning test: big
1322026-07-28T04:38:58.581ZCompleted test: big
1332026-07-28T04:39:03.587ZRunning test: dep
1342026-07-28T04:39:22.818ZCompleted test: dep
1352026-07-28T04:39:27.823ZRunning test: balloon
1362026-07-28T04:41:49.617ZCompleted test: balloon
1372026-07-28T04:41:54.622ZRunning test: deactivate
1382026-07-28T04:41:58.946ZCompleted test: deactivate
1392026-07-28T04:42:03.952ZRunning hammer
1402026-07-28T04:42:10.821ZRun repair tests
1412026-07-28T04:42:10.824Z/var/tmp/bins/crutest fill -g 31 -q --key 3nhxQym94pYVbi5RDOWrNoPrbwhDkUuuDP5Cs9MKUFc= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1422026-07-28T04:42:10.842Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.309892959Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"task":"crutest"}
1432026-07-28T04:42:10.847Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.310194619Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"task":"crutest"}
1442026-07-28T04:42:10.847Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.310768932Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"task":"crutest"}
1452026-07-28T04:42:10.847Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.311001849Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182}
1462026-07-28T04:42:10.847Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.311033506Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182}
1472026-07-28T04:42:10.847Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.311042955Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182}
1482026-07-28T04:42:10.847Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.311055624Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182}
1492026-07-28T04:42:10.847Z{"msg":"Crucible 11e26fec-cce3-44e3-8572-3bb47baba03c has session id: 37640fbc-4925-49c7-870e-0a278e2c89d2","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.311427667Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1502026-07-28T04:42:10.847Z{"msg":"Upstairs opts: Upstairs UUID: 11e26fec-cce3-44e3-8572-3bb47baba03c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.311453315Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1512026-07-28T04:42:10.847Z{"msg":"Crucible stats registered with UUID: 11e26fec-cce3-44e3-8572-3bb47baba03c","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.311464404Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1522026-07-28T04:42:10.847Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:09.311479102Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1532026-07-28T04:42:10.847Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.311645516Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1542026-07-28T04:42:10.847Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.311679182Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1552026-07-28T04:42:10.848Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.311691311Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1562026-07-28T04:42:10.848Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.312761425Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"task":"crutest"}
1572026-07-28T04:42:10.848Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.31281441Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182}
1582026-07-28T04:42:10.848Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.312836818Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182}
1592026-07-28T04:42:10.848Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.312846657Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182}
1602026-07-28T04:42:10.848Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.312859036Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182}
1612026-07-28T04:42:10.848Z{"msg":"Crucible 11e26fec-cce3-44e3-8572-3bb47baba03c has session id: 24bed490-4a16-4312-81af-c87f48177fea","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.31301505Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
1622026-07-28T04:42:10.848Z{"msg":"Upstairs opts: Upstairs UUID: 11e26fec-cce3-44e3-8572-3bb47baba03c, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.313031339Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
1632026-07-28T04:42:10.848Z{"msg":"Crucible stats registered with UUID: 11e26fec-cce3-44e3-8572-3bb47baba03c","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.313042358Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
1642026-07-28T04:42:10.848Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:09.313060316Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
1652026-07-28T04:42:10.848Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.313070565Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
1662026-07-28T04:42:10.848Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.313082614Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
1672026-07-28T04:42:10.848Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.313093673Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
1682026-07-28T04:42:10.848Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.31321879Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"task":"crutest"}
1692026-07-28T04:42:10.848Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.313233829Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182}
1702026-07-28T04:42:10.848Z{"msg":"11e26fec-cce3-44e3-8572-3bb47baba03c active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.313247987Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1712026-07-28T04:42:10.848Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.313257516Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"client":"0","":"downstairs","session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1722026-07-28T04:42:10.848Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.313268865Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"client":"1","":"downstairs","session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1732026-07-28T04:42:10.848Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.313279314Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"client":"2","":"downstairs","session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1742026-07-28T04:42:10.848Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.313289403Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1752026-07-28T04:42:10.848Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.313304902Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1762026-07-28T04:42:10.848Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.313315571Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1772026-07-28T04:42:10.848Z{"msg":"ds_connection connected from Ok(127.0.0.1:53092)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.313356507Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1782026-07-28T04:42:10.848Z{"msg":"ds_connection connected from Ok(127.0.0.1:37473)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.313381574Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1792026-07-28T04:42:10.848Z{"msg":"ds_connection connected from Ok(127.0.0.1:62236)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.313511631Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1802026-07-28T04:42:10.853Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314043429Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"client":"0","":"downstairs","session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1812026-07-28T04:42:10.853Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314070366Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"client":"1","":"downstairs","session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1822026-07-28T04:42:10.853Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314083375Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"client":"2","":"downstairs","session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1832026-07-28T04:42:10.853Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314224171Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1842026-07-28T04:42:10.853Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314248808Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1852026-07-28T04:42:10.853Z{"msg":"[0]R flush_numbers: [4000, 4000, 4000, 4000, 4000]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314318112Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"downstairs","":"downstairs","session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1862026-07-28T04:42:10.853Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.31433031Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"downstairs","":"downstairs","session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1872026-07-28T04:42:10.853Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314341839Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"downstairs","":"downstairs","session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1882026-07-28T04:42:10.853Z{"msg":"[1]R flush_numbers: [4000, 4000, 4000, 4000, 4000]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314353108Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"downstairs","":"downstairs","session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1892026-07-28T04:42:10.853Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314364097Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"downstairs","":"downstairs","session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1902026-07-28T04:42:10.854Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314375326Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"downstairs","":"downstairs","session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1912026-07-28T04:42:10.854Z{"msg":"[2]R flush_numbers: [4000, 4000, 4000, 4000, 4000]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314386025Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"downstairs","":"downstairs","session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1922026-07-28T04:42:10.854Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314396764Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"downstairs","":"downstairs","session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1932026-07-28T04:42:10.854Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314407723Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"downstairs","":"downstairs","session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1942026-07-28T04:42:10.854Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314418052Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"downstairs","":"downstairs","session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1952026-07-28T04:42:10.854Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314428721Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"downstairs","":"downstairs","session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1962026-07-28T04:42:10.854Z{"msg":"Next flush: 4001","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314439Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"downstairs","":"downstairs","session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1972026-07-28T04:42:10.854Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314449809Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"downstairs","":"downstairs","session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1982026-07-28T04:42:10.854Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314460477Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
1992026-07-28T04:42:10.854Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314470107Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"client":"0","":"downstairs","session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
2002026-07-28T04:42:10.854Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314480705Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"client":"1","":"downstairs","session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
2012026-07-28T04:42:10.854Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314491244Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"client":"2","":"downstairs","session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
2022026-07-28T04:42:10.854Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314507213Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
2032026-07-28T04:42:10.854Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314517002Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
2042026-07-28T04:42:10.854Z{"msg":"11e26fec-cce3-44e3-8572-3bb47baba03c is now active with session: 37640fbc-4925-49c7-870e-0a278e2c89d2","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.31453201Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
2052026-07-28T04:42:10.854Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314541829Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"session_id":"37640fbc-4925-49c7-870e-0a278e2c89d2"}
2062026-07-28T04:42:10.854Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314580626Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182}
2072026-07-28T04:42:10.854Z{"msg":"Activated sub_volume 11e26fec-cce3-44e3-8572-3bb47baba03c","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314595174Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182}
2082026-07-28T04:42:10.854Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314603913Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182}
2092026-07-28T04:42:10.854Z{"msg":"11e26fec-cce3-44e3-8572-3bb47baba03c active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314612502Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2102026-07-28T04:42:10.854Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314622151Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"client":"0","":"downstairs","session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2112026-07-28T04:42:10.854Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314653778Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"client":"1","":"downstairs","session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2122026-07-28T04:42:10.854Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314666337Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"client":"2","":"downstairs","session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2132026-07-28T04:42:10.854Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314689865Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2142026-07-28T04:42:10.854Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314705033Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2152026-07-28T04:42:10.854Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314715602Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2162026-07-28T04:42:10.854Z{"msg":"ds_connection connected from Ok(127.0.0.1:58198)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314726301Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2172026-07-28T04:42:10.855Z{"msg":"ds_connection connected from Ok(127.0.0.1:54140)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.31473768Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2182026-07-28T04:42:10.855Z{"msg":"ds_connection connected from Ok(127.0.0.1:39200)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314748949Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2192026-07-28T04:42:10.855Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.31493993Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"client":"2","":"downstairs","session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2202026-07-28T04:42:10.855Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314972917Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"client":"1","":"downstairs","session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2212026-07-28T04:42:10.855Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.314985636Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"client":"0","":"downstairs","session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2222026-07-28T04:42:10.855Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.315112293Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2232026-07-28T04:42:10.855Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.31514174Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2242026-07-28T04:42:10.855Z{"msg":"[0]R flush_numbers: [5320, 5320, 5319, 5317, 5313]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.315160578Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"downstairs","":"downstairs","session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2252026-07-28T04:42:10.855Z{"msg":"[0]R generation: [14, 14, 14, 13, 12]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.315175937Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"downstairs","":"downstairs","session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2262026-07-28T04:42:10.855Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.315186746Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"downstairs","":"downstairs","session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2272026-07-28T04:42:10.855Z{"msg":"[1]R flush_numbers: [5320, 5320, 5319, 5317, 5313]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.315201294Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"downstairs","":"downstairs","session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2282026-07-28T04:42:10.855Z{"msg":"[1]R generation: [14, 14, 14, 13, 12]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.315212133Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"downstairs","":"downstairs","session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2292026-07-28T04:42:10.855Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.315226822Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"downstairs","":"downstairs","session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2302026-07-28T04:42:10.855Z{"msg":"[2]R flush_numbers: [5320, 5320, 5319, 5317, 5313]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.315238031Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"downstairs","":"downstairs","session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2312026-07-28T04:42:10.855Z{"msg":"[2]R generation: [14, 14, 14, 13, 12]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.315252429Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"downstairs","":"downstairs","session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2322026-07-28T04:42:10.855Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.315267318Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"downstairs","":"downstairs","session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2332026-07-28T04:42:10.855Z{"msg":"Max found gen is 15","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.315281136Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"downstairs","":"downstairs","session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2342026-07-28T04:42:10.855Z{"msg":"Generation requested: 31 >= found:15","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.315291905Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"downstairs","":"downstairs","session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2352026-07-28T04:42:10.855Z{"msg":"Next flush: 5321","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.315305894Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"downstairs","":"downstairs","session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2362026-07-28T04:42:10.855Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.315316573Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"":"downstairs","":"downstairs","session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2372026-07-28T04:42:10.855Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.315331051Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2382026-07-28T04:42:10.855Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.31534086Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"client":"0","":"downstairs","session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2392026-07-28T04:42:10.855Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.315355419Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"client":"1","":"downstairs","session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2402026-07-28T04:42:10.855Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.315366028Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"client":"2","":"downstairs","session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2412026-07-28T04:42:10.855Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.315380446Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2422026-07-28T04:42:10.855Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.315390605Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"session_id":"Disk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
2432026-07-28T04:42:10.855Z24bed490-4a16-4312-81af-c87f48177fea"}
2442026-07-28T04:42:10.856Z{"msg":"11e26fec-cce3-44e3-8572-3bb47baba03c is now active with session: 24bed490-4a16-4312-81af-c87f48177fea","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.315410823Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2452026-07-28T04:42:10.856Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.315420013Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"session_id":"24bed490-4a16-4312-81af-c87f48177fea"}
2462026-07-28T04:42:10.856Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.315434111Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182}
2472026-07-28T04:42:10.856Z{"msg":"Activated sub_volume 11e26fec-cce3-44e3-8572-3bb47baba03c","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.31544339Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182}
2482026-07-28T04:42:10.856Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.315456549Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"task":"crutest"}
2492026-07-28T04:42:10.857Z{"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.315465078Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"task":"crutest"}
2502026-07-28T04:42:10.871ZRead and Verify all blocks (0..100 range:false)
2512026-07-28T04:42:10.918Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.385785705Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"task":"crutest"}
2522026-07-28T04:42:10.918Z----------------------------------------------------------------
2532026-07-28T04:42:10.918Z{" Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2542026-07-28T04:42:10.918Zmsg"States::" Active Active ActiveCLIENT: Tests done. All submitted work has been ACK'd
2552026-07-28T04:42:10.918Z","v":0,"name":"crucible","level":30JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2562026-07-28T04:42:10.918Z,"time":"2026-07-28T04:42:09.385814682Z","hostname":" 1002 Acked Read 50w-01KYKG6JQGQ8Z9NT39M0RRXVEB"," Donepid": Done1182 Done false
2572026-07-28T04:42:10.922Z,"task":"crutest"}
2582026-07-28T04:42:10.922Z STATES DS:0 DS:1 DS:2 TOTAL
2592026-07-28T04:42:10.922Z Sent 0 0 0 0
2602026-07-28T04:42:10.922Z Done 1 1 1 3
2612026-07-28T04:42:10.922Z Skipped 0 0 0 0
2622026-07-28T04:42:10.922Z Error 0 0 0 0
2632026-07-28T04:42:10.922ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2642026-07-28T04:42:10.922ZDownstairs last five completed: 1001 1000
2652026-07-28T04:42:10.922ZUpstairs last five completed: 1002 1001 1000
2662026-07-28T04:42:10.922Z----------------------------------------------------------------
2672026-07-28T04:42:10.922Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2682026-07-28T04:42:10.922ZStates: Active Active Active
2692026-07-28T04:42:10.922ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2702026-07-28T04:42:10.922Z 1002 Acked Read 50 Done Done Done false
2712026-07-28T04:42:10.922Z STATES DS:0 DS:1 DS:2 TOTAL
2722026-07-28T04:42:10.922Z Sent 0 0 0 0
2732026-07-28T04:42:10.922Z Done 1 1 1 3
2742026-07-28T04:42:10.922Z Skipped 0 0 0 0
2752026-07-28T04:42:10.922Z Error 0 0 0 0
2762026-07-28T04:42:10.922ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2772026-07-28T04:42:10.922ZDownstairs last five completed: 1001 1000
2782026-07-28T04:42:10.922ZUpstairs last five completed: 1002 1001 1000
2792026-07-28T04:42:10.922Z{"msg":"CLIENT: Up:0 ds:2 act:6","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:09.385909402Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"task":"crutest"}
2802026-07-28T04:42:14.920Z----------------------------------------------------------------
2812026-07-28T04:42:14.920Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2822026-07-28T04:42:14.920ZDownstairs last five completed: 1003 1002 1001 1000
2832026-07-28T04:42:14.920ZUpstairs last five completed: 1003 1002 1001 1000
2842026-07-28T04:42:14.923Z----------------------------------------------------------------
2852026-07-28T04:42:14.923Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2862026-07-28T04:42:14.923ZDownstairs last five completed: 1003 1002 1001 1000
2872026-07-28T04:42:14.923ZUpstairs last five completed: 1003 1002 1001 1000
2882026-07-28T04:42:14.923Z{"msg":"CLIENT: Up:0 ds:0 act:6","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.387462963Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"task":"crutest"}
2892026-07-28T04:42:14.923Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.387509248Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1182,"task":"crutest"}
2902026-07-28T04:42:14.928ZRepair setup passed
2912026-07-28T04:42:14.931ZCopy the region for /var/tmp/test_up-build/8830
2922026-07-28T04:42:14.931Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2932026-07-28T04:42:14.934Z/var/tmp/bins/crutest fill -g 32 -q --key 3nhxQym94pYVbi5RDOWrNoPrbwhDkUuuDP5Cs9MKUFc= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2942026-07-28T04:42:14.952Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.419216863Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"task":"crutest"}
2952026-07-28T04:42:14.955Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.419702315Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"task":"crutest"}
2962026-07-28T04:42:14.955Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.420490137Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"task":"crutest"}
2972026-07-28T04:42:14.955Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.420723154Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188}
2982026-07-28T04:42:14.955Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.420751181Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188}
2992026-07-28T04:42:14.955Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.42076468Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188}
3002026-07-28T04:42:14.955Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.420777518Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188}
3012026-07-28T04:42:14.955Z{"msg":"Crucible 12f7f083-8a1e-44d9-9a29-9de913e25ec7 has session id: 0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.421184298Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3022026-07-28T04:42:14.955Z{"msg":"Upstairs opts: Upstairs UUID: 12f7f083-8a1e-44d9-9a29-9de913e25ec7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.421225144Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3032026-07-28T04:42:14.955Z{"msg":"Crucible stats registered with UUID: 12f7f083-8a1e-44d9-9a29-9de913e25ec7","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.421241293Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3042026-07-28T04:42:14.955Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:13.421251212Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3052026-07-28T04:42:14.955Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.421394797Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3062026-07-28T04:42:14.955Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.421432584Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3072026-07-28T04:42:14.955Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.421452922Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3082026-07-28T04:42:14.955Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.422404938Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"task":"crutest"}
3092026-07-28T04:42:14.955Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.422471051Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188}
3102026-07-28T04:42:14.955Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.422496738Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188}
3112026-07-28T04:42:14.955Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.422510117Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188}
3122026-07-28T04:42:14.955Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.422522356Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188}
3132026-07-28T04:42:14.960Z{"msg":"Crucible 12f7f083-8a1e-44d9-9a29-9de913e25ec7 has session id: 9c57ffe1-737e-4207-8c1b-2b5a77bbc861","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.422722796Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3142026-07-28T04:42:14.960Z{"msg":"Upstairs opts: Upstairs UUID: 12f7f083-8a1e-44d9-9a29-9de913e25ec7, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.422742524Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3152026-07-28T04:42:14.960Z{"msg":"Crucible stats registered with UUID: 12f7f083-8a1e-44d9-9a29-9de913e25ec7","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.422753603Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3162026-07-28T04:42:14.960Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:13.422767192Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3172026-07-28T04:42:14.960Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.422803988Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3182026-07-28T04:42:14.960Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.422834495Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3192026-07-28T04:42:14.960Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.422845744Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3202026-07-28T04:42:14.960Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.422965202Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"task":"crutest"}
3212026-07-28T04:42:14.960Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.42298352Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188}
3222026-07-28T04:42:14.960Z{"msg":"12f7f083-8a1e-44d9-9a29-9de913e25ec7 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.422995019Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3232026-07-28T04:42:14.960Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.423014607Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"client":"0","":"downstairs","session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3242026-07-28T04:42:14.960Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.423029526Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"client":"1","":"downstairs","session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3252026-07-28T04:42:14.960Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.423039835Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"client":"2","":"downstairs","session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3262026-07-28T04:42:14.960Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.42308419Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3272026-07-28T04:42:14.960Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.423094779Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3282026-07-28T04:42:14.960Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.423113138Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3292026-07-28T04:42:14.961Z{"msg":"ds_connection connected from Ok(127.0.0.1:50873)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.42318792Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3302026-07-28T04:42:14.961Z{"msg":"ds_connection connected from Ok(127.0.0.1:39123)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.423203609Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3312026-07-28T04:42:14.961Z{"msg":"ds_connection connected from Ok(127.0.0.1:64934)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.423213998Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3322026-07-28T04:42:14.961Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.4238937Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"client":"0","":"downstairs","session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3332026-07-28T04:42:14.961Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.423936976Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"client":"1","":"downstairs","session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3342026-07-28T04:42:14.961Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424053095Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"client":"2","":"downstairs","session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3352026-07-28T04:42:14.961Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424133767Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3362026-07-28T04:42:14.961Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424149055Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3372026-07-28T04:42:14.961Z{"msg":"[0]R flush_numbers: [4001, 4001, 4001, 4001, 4001]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424205469Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"downstairs","":"downstairs","session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3382026-07-28T04:42:14.961Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424221038Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"downstairs","":"downstairs","session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3392026-07-28T04:42:14.961Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424231437Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"downstairs","":"downstairs","session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3402026-07-28T04:42:14.961Z{"msg":"[1]R flush_numbers: [4001, 4001, 4001, 4001, 4001]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424246145Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"downstairs","":"downstairs","session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3412026-07-28T04:42:14.961Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424256514Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"downstairs","":"downstairs","session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3422026-07-28T04:42:14.961Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424271053Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"downstairs","":"downstairs","session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3432026-07-28T04:42:14.963Z{"msg":"[2]R flush_numbers: [4001, 4001, 4001, 4001, 4001]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424281412Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"downstairs","":"downstairs","session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3442026-07-28T04:42:14.963Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424292051Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"downstairs","":"downstairs","session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3452026-07-28T04:42:14.963Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.42430271Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"downstairs","":"downstairs","session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3462026-07-28T04:42:14.963Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424313849Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"downstairs","":"downstairs","session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3472026-07-28T04:42:14.963Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424324898Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"downstairs","":"downstairs","session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3482026-07-28T04:42:14.963Z{"msg":"Next flush: 4002","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424335527Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"downstairs","":"downstairs","session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3492026-07-28T04:42:14.963Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424345966Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"downstairs","":"downstairs","session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3502026-07-28T04:42:14.963Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424356995Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3512026-07-28T04:42:14.963Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424367223Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"client":"0","":"downstairs","session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3522026-07-28T04:42:14.963Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424377812Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"client":"1","":"downstairs","session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3532026-07-28T04:42:14.963Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424387691Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"client":"2","":"downstairs","session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3542026-07-28T04:42:14.963Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.42439779Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3552026-07-28T04:42:14.963Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424412439Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3562026-07-28T04:42:14.963Z{"msg":"12f7f083-8a1e-44d9-9a29-9de913e25ec7 is now active with session: 0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424423168Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3572026-07-28T04:42:14.963Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424440206Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"session_id":"0ae4f6f3-5957-4fbb-9a07-d07c4a7592d6"}
3582026-07-28T04:42:14.963Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424450305Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188}
3592026-07-28T04:42:14.963Z{"msg":"Activated sub_volume 12f7f083-8a1e-44d9-9a29-9de913e25ec7","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424459224Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188}
3602026-07-28T04:42:14.963Z{"msg":"12f7f083-8a1e-44d9-9a29-9de913e25ec7 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424468004Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3612026-07-28T04:42:14.963Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424477573Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188}
3622026-07-28T04:42:14.963Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424485892Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"client":"0","":"downstairs","session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3632026-07-28T04:42:14.963Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424495791Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"client":"1","":"downstairs","session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3642026-07-28T04:42:14.963Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.42450549Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"client":"2","":"downstairs","session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3652026-07-28T04:42:14.963Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424515619Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3662026-07-28T04:42:14.963Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424554475Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3672026-07-28T04:42:14.964Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424565184Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3682026-07-28T04:42:14.964Z{"msg":"ds_connection connected from Ok(127.0.0.1:59066)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424575783Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3692026-07-28T04:42:14.964Z{"msg":"ds_connection connected from Ok(127.0.0.1:49854)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424587002Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3702026-07-28T04:42:14.964Z{"msg":"ds_connection connected from Ok(127.0.0.1:60420)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424597751Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3712026-07-28T04:42:14.964Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424868694Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"client":"0","":"downstairs","session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3722026-07-28T04:42:14.964Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424924278Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"client":"2","":"downstairs","session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3732026-07-28T04:42:14.964Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.424940567Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"client":"1","":"downstairs","session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3742026-07-28T04:42:14.964Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.425028618Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3752026-07-28T04:42:14.964Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.42510702Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3762026-07-28T04:42:14.964Z{"msg":"[0]R flush_numbers: [5321, 5321, 5321, 5321, 5321]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.425145427Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"downstairs","":"downstairs","session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3772026-07-28T04:42:14.964Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.425160785Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"downstairs","":"downstairs","session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3782026-07-28T04:42:14.964Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.425171064Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"downstairs","":"downstairs","session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3792026-07-28T04:42:14.964Z{"msg":"[1]R flush_numbers: [5321, 5321, 5321, 5321, 5321]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.425185793Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"downstairs","":"downstairs","session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3802026-07-28T04:42:14.964Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.425196112Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"downstairs","":"downstairs","session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3812026-07-28T04:42:14.964Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.42521083Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"downstairs","":"downstairs","session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3822026-07-28T04:42:14.965Z{"msg":"[2]R flush_numbers: [5321, 5321, 5321, 5321, 5321]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.425221159Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"downstairs","":"downstairs","session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3832026-07-28T04:42:14.965Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.425235548Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"downstairs","":"downstairs","session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3842026-07-28T04:42:14.965Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.425245767Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"downstairs","":"downstairs","session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3852026-07-28T04:42:14.965Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.425259745Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"downstairs","":"downstairs","session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3862026-07-28T04:42:14.965Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.425270184Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"downstairs","":"downstairs","session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3872026-07-28T04:42:14.965Z{"msg":"Next flush: 5322","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.425284153Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"downstairs","":"downstairs","session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3882026-07-28T04:42:14.965Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.425294292Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"":"downstairs","":"downstairs","session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3892026-07-28T04:42:14.965Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.42530909Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3902026-07-28T04:42:14.965Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.425318629Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"client":"0","":"downstairs","session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3912026-07-28T04:42:14.965Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.425332958Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"client":"1","":"downstairs","session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3922026-07-28T04:42:14.965Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.425342907Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"client":"2","":"downstairs","session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3932026-07-28T04:42:14.965Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.425357496Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3942026-07-28T04:42:14.965Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.425372134Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3952026-07-28T04:42:14.965Z{"msg":"12f7f083-8a1e-44d9-9a29-9de913e25ec7 is now active with session: 9c57ffe1-737e-4207-8c1b-2b5a77bbc861","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.425387023Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3962026-07-28T04:42:14.965Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.425396022Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"session_id":"9c57ffe1-737e-4207-8c1b-2b5a77bbc861"}
3972026-07-28T04:42:14.965Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.4254102Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188}
3982026-07-28T04:42:14.965Z{"msg":"Activated sub_volume 12f7f083-8a1e-44d9-9a29-9de913e25ec7","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.425419289Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188}
3992026-07-28T04:42:14.965Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.425432458Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"task":"crutest"}
4002026-07-28T04:42:14.965ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
4012026-07-28T04:42:14.965ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4022026-07-28T04:42:14.965Z{"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.425785183Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"task":"crutest"}
4032026-07-28T04:42:14.982ZRead and Verify all blocks (0..100 range:false)
4042026-07-28T04:42:15.028Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30----------------------------------------------------------------
4052026-07-28T04:42:15.028Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4062026-07-28T04:42:15.028Z,States:"time":" Active2026-07-28T04:42:13.495370042Z Active" Active
4072026-07-28T04:42:15.028Z,"hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid"JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4082026-07-28T04:42:15.028Z:1188,"task":"crutest" 1002 Acked Read 50}
4092026-07-28T04:42:15.028Z Done Done Done false
4102026-07-28T04:42:15.028Z{"msg":"CLIENT: Tests done. All submitted work has been ACK'd","v":0,"name":"crucible","level":30 STATES DS:0 DS:1 DS:2 TOTAL
4112026-07-28T04:42:15.028Z Sent 0 0 0 0
4122026-07-28T04:42:15.028Z," Done 1 time" 1 : 1 " 3
4132026-07-28T04:42:15.028Z2026-07-28T04:42:13.495400769Z" Skipped , 0 " 0 0 0
4142026-07-28T04:42:15.028Zhostname":" Error 0 0 0 0
4152026-07-28T04:42:15.028Zw-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":Last Flush: 1188Some(JobId(1001)) ,Some(JobId(1001)) "Some(JobId(1001)) task"
4162026-07-28T04:42:15.028Z:Downstairs last five completed:"crutest"}
4172026-07-28T04:42:15.028Z 1001 1000
4182026-07-28T04:42:15.028ZUpstairs last five completed: 1002 1001 1000
4192026-07-28T04:42:15.032Z----------------------------------------------------------------
4202026-07-28T04:42:15.032Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4212026-07-28T04:42:15.032ZStates: Active Active Active
4222026-07-28T04:42:15.032ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4232026-07-28T04:42:15.032Z 1002 Acked Read 50 Done Done Done false
4242026-07-28T04:42:15.032Z STATES DS:0 DS:1 DS:2 TOTAL
4252026-07-28T04:42:15.032Z Sent 0 0 0 0
4262026-07-28T04:42:15.032Z Done 1 1 1 3
4272026-07-28T04:42:15.032Z Skipped 0 0 0 0
4282026-07-28T04:42:15.032Z Error 0 0 0 0
4292026-07-28T04:42:15.032ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4302026-07-28T04:42:15.032ZDownstairs last five completed: 1001 1000
4312026-07-28T04:42:15.032ZUpstairs last five completed: 1002 1001 1000
4322026-07-28T04:42:15.032Z{"msg":"CLIENT: Up:0 ds:2 act:6","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:13.495528297Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"task":"crutest"}
4332026-07-28T04:42:19.030Z----------------------------------------------------------------
4342026-07-28T04:42:19.030Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4352026-07-28T04:42:19.030ZDownstairs last five completed: 1003 1002 1001 1000
4362026-07-28T04:42:19.030ZUpstairs last five completed: 1003 1002 1001 1000
4372026-07-28T04:42:19.030Z----------------------------------------------------------------
4382026-07-28T04:42:19.030Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4392026-07-28T04:42:19.030ZDownstairs last five completed: 1003 1002 1001 1000
4402026-07-28T04:42:19.030ZUpstairs last five completed: 1003 1002 1001 1000
4412026-07-28T04:42:19.034Z{"msg":"CLIENT: Up:0 ds:0 act:6","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.496906995Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"task":"crutest"}
4422026-07-28T04:42:19.034Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.496969738Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1188,"task":"crutest"}
4432026-07-28T04:42:19.037ZRepair part 1 passed
4442026-07-28T04:42:19.037Z
4452026-07-28T04:42:19.037ZKill the current downstairs
4462026-07-28T04:42:19.079ZDownstairs 2 stopped
4472026-07-28T04:42:19.079Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
4482026-07-28T04:42:19.082ZNow put back the original so we have a mismatch
4492026-07-28T04:42:19.082Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
4502026-07-28T04:42:19.082ZRestart downstairs with old directory
4512026-07-28T04:42:19.095Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4522026-07-28T04:42:19.107ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4532026-07-28T04:42:19.110Z 0 000-009 32 32 31 4002 4002 4001 F F F <---
4542026-07-28T04:42:19.110Z 1 010-019 32 32 31 4002 4002 4001 F F F <---
4552026-07-28T04:42:19.110Z 2 020-029 32 32 31 4002 4002 4001 F F F <---
4562026-07-28T04:42:19.110Z 3 030-039 32 32 31 4002 4002 4001 F F F <---
4572026-07-28T04:42:19.110Z 4 040-049 32 32 31 4002 4002 4001 F F F <---
4582026-07-28T04:42:19.110ZMax gen: 32, Max flush: 4002
4592026-07-28T04:42:19.110ZError: Difference in extent metadata found!
4602026-07-28T04:42:19.110Zdump test found error as expected
4612026-07-28T04:42:19.110Z
4622026-07-28T04:42:19.111Z
4632026-07-28T04:42:19.111Z/var/tmp/bins/crutest verify --range -g 33 -q --key 3nhxQym94pYVbi5RDOWrNoPrbwhDkUuuDP5Cs9MKUFc= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
4642026-07-28T04:42:19.128Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.5950448Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"task":"crutest"}
4652026-07-28T04:42:19.131Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.595290436Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"task":"crutest"}
4662026-07-28T04:42:19.131Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.595899906Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"task":"crutest"}
4672026-07-28T04:42:19.131Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.596129963Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196}
4682026-07-28T04:42:19.131Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.59616298Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196}
4692026-07-28T04:42:19.131Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.596172959Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196}
4702026-07-28T04:42:19.131Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.596185468Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196}
4712026-07-28T04:42:19.131Z{"msg":"Crucible 5ee40cce-78ea-4fa9-858d-00f9382f8e48 has session id: 01811005-af3b-403d-94ba-f727125f0e14","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.59656058Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
4722026-07-28T04:42:19.131Z{"msg":"Upstairs opts: Upstairs UUID: 5ee40cce-78ea-4fa9-858d-00f9382f8e48, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.596600876Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
4732026-07-28T04:42:19.131Z{"msg":"Crucible stats registered with UUID: 5ee40cce-78ea-4fa9-858d-00f9382f8e48","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.596613705Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
4742026-07-28T04:42:19.131Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:17.596624144Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
4752026-07-28T04:42:19.131Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.596754001Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
4762026-07-28T04:42:19.131Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.596785298Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
4772026-07-28T04:42:19.131Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.596797557Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
4782026-07-28T04:42:19.131Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.597558682Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"task":"crutest"}
4792026-07-28T04:42:19.131Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.597609857Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196}
4802026-07-28T04:42:19.131Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.597630915Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196}
4812026-07-28T04:42:19.131Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.597640114Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196}
4822026-07-28T04:42:19.131Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.597652642Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196}
4832026-07-28T04:42:19.131Z{"msg":"Crucible 5ee40cce-78ea-4fa9-858d-00f9382f8e48 has session id: 0511d689-1997-4ca4-8410-2e41d94f8431","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.59787667Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
4842026-07-28T04:42:19.131Z{"msg":"Upstairs opts: Upstairs UUID: 5ee40cce-78ea-4fa9-858d-00f9382f8e48, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.597893469Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
4852026-07-28T04:42:19.132Z{"msg":"Crucible stats registered with UUID: 5ee40cce-78ea-4fa9-858d-00f9382f8e48","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.597904648Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
4862026-07-28T04:42:19.132Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:17.597914447Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
4872026-07-28T04:42:19.132Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.597941334Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
4882026-07-28T04:42:19.132Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.597957602Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
4892026-07-28T04:42:19.132Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.597968591Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
4902026-07-28T04:42:19.132Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.59807933Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"task":"crutest"}
4912026-07-28T04:42:19.132Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.598094739Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196}
4922026-07-28T04:42:19.132Z{"msg":"5ee40cce-78ea-4fa9-858d-00f9382f8e48 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.598125706Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
4932026-07-28T04:42:19.132Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.598136295Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"client":"0","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
4942026-07-28T04:42:19.132Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.598147184Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"client":"1","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
4952026-07-28T04:42:19.132Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.598157413Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"client":"2","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
4962026-07-28T04:42:19.132Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.598167871Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
4972026-07-28T04:42:19.132Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.59817897Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
4982026-07-28T04:42:19.132Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.598189949Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
4992026-07-28T04:42:19.132Z{"msg":"ds_connection connected from Ok(127.0.0.1:60280)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.598243164Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5002026-07-28T04:42:19.132Z{"msg":"ds_connection connected from Ok(127.0.0.1:41196)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.598258852Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5012026-07-28T04:42:19.132Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:17.598271091Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5022026-07-28T04:42:19.132Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:17.59828404Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5032026-07-28T04:42:19.132Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:17.598297259Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5042026-07-28T04:42:19.135Z{"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.598400498Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5052026-07-28T04:42:19.135Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.598423316Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5062026-07-28T04:42:19.135Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.598914488Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"client":"1","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5072026-07-28T04:42:19.135Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.598943465Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"client":"0","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5082026-07-28T04:42:19.135Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.599083011Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5092026-07-28T04:42:19.135Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:17.59909616Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5102026-07-28T04:42:29.133Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.599415508Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5112026-07-28T04:42:29.133Z{"msg":"ds_connection connected from Ok(127.0.0.1:42368)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.599491441Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5122026-07-28T04:42:29.139Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.600803991Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"client":"2","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5132026-07-28T04:42:29.139Z{"msg":"[0]R flush_numbers: [4002, 4002, 4002, 4002, 4002]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.601080243Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"downstairs","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5142026-07-28T04:42:29.139Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.601104141Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"downstairs","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5152026-07-28T04:42:29.139Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.60111961Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"downstairs","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5162026-07-28T04:42:29.139Z{"msg":"[1]R flush_numbers: [4002, 4002, 4002, 4002, 4002]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.601142277Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"downstairs","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5172026-07-28T04:42:29.139Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.601157406Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"downstairs","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5182026-07-28T04:42:29.139Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.601178774Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"downstairs","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5192026-07-28T04:42:29.139Z{"msg":"[2]R flush_numbers: [4001, 4001, 4001, 4001, 4001]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.601193992Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"downstairs","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5202026-07-28T04:42:29.140Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.601208411Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"downstairs","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5212026-07-28T04:42:29.140Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.601223849Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"downstairs","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5222026-07-28T04:42:29.140Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.601244287Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"downstairs","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5232026-07-28T04:42:29.140Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.601259206Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"downstairs","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5242026-07-28T04:42:29.140Z{"msg":"Next flush: 4003","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.601279624Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"downstairs","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5252026-07-28T04:42:29.140Z{"msg":"extent 0 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4002, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4002, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4001, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.601300302Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5262026-07-28T04:42:29.140Z{"msg":"extent:0 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4002, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4002, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4001, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.601328239Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5272026-07-28T04:42:29.140Z{"msg":"found dest for source 0 for extent at index 0 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.601350737Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5282026-07-28T04:42:29.140Z{"msg":"extent 1 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4002, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4002, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4001, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.601371425Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5292026-07-28T04:42:29.141Z{"msg":"extent:1 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4002, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4002, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4001, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.601398372Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5302026-07-28T04:42:29.141Z{"msg":"found dest for source 0 for extent at index 1 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.60142051Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5312026-07-28T04:42:29.141Z{"msg":"extent 2 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4002, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4002, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4001, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.601441548Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5322026-07-28T04:42:29.142Z{"msg":"extent:2 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4002, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4002, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4001, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.601469105Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5332026-07-28T04:42:29.142Z{"msg":"found dest for source 0 for extent at index 2 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.601485993Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5342026-07-28T04:42:29.142Z{"msg":"extent 3 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4002, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4002, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4001, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.601506521Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5352026-07-28T04:42:29.142Z{"msg":"extent:3 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4002, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4002, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4001, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.601533429Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5362026-07-28T04:42:29.142Z{"msg":"found dest for source 0 for extent at index 3 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.601555307Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5372026-07-28T04:42:29.142Z{"msg":"extent 4 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4002, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4002, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4001, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.601575834Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5382026-07-28T04:42:29.142Z{"msg":"extent:4 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4002, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4002, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4001, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.601602212Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5392026-07-28T04:42:29.142Z{"msg":"found dest for source 0 for extent at index 4 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.60162408Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5402026-07-28T04:42:29.142Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.601641698Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"client":"0","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5412026-07-28T04:42:29.142Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.601658156Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"client":"1","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5422026-07-28T04:42:29.142Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.601673545Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"client":"2","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5432026-07-28T04:42:29.142Z{"msg":"Full repair list: {ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.601713551Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"downstairs","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5442026-07-28T04:42:29.143Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 4003, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 4003, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 4003, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 4003, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 4003, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.601851777Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"downstairs","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5452026-07-28T04:42:29.145Z{"msg":"starting reconciliation 539f8c26-c574-4f87-88f4-51e638288479: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.601868456Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"downstairs","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5462026-07-28T04:42:29.145Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.601883244Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"client":"2","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5472026-07-28T04:42:29.149Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.614203516Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"client":"2","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5482026-07-28T04:42:29.152Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.616330006Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"client":"2","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5492026-07-28T04:42:29.155Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.618331528Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"client":"2","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5502026-07-28T04:42:29.155Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.620285824Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"client":"2","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5512026-07-28T04:42:29.158Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.622011164Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"downstairs","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5522026-07-28T04:42:29.158Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.622026892Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"client":"0","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5532026-07-28T04:42:29.159Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.622037331Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"client":"1","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5542026-07-28T04:42:29.159Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.62205184Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"client":"2","":"downstairs","session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5552026-07-28T04:42:29.159Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.622062599Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5562026-07-28T04:42:29.159Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.622076807Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5572026-07-28T04:42:29.159Z{"msg":"5ee40cce-78ea-4fa9-858d-00f9382f8e48 is now active with session: 01811005-af3b-403d-94ba-f727125f0e14","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.622087946Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"session_id":"01811005-af3b-403d-94ba-f727125f0e14"}
5582026-07-28T04:42:29.159Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.622101835Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196}
5592026-07-28T04:42:29.159Z{"msg":"Activated sub_volume 5ee40cce-78ea-4fa9-858d-00f9382f8e48","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.62214768Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196}
5602026-07-28T04:42:29.159Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.622210274Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196}
5612026-07-28T04:42:29.159Z{"msg":"5ee40cce-78ea-4fa9-858d-00f9382f8e48 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.622223763Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5622026-07-28T04:42:29.159Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.622233642Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"client":"0","":"downstairs","session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5632026-07-28T04:42:29.159Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.62224844Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"client":"1","":"downstairs","session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5642026-07-28T04:42:29.159Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.622263179Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"client":"2","":"downstairs","session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5652026-07-28T04:42:29.159Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.622273538Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5662026-07-28T04:42:29.159Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.622284167Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5672026-07-28T04:42:29.159Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.622294666Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5682026-07-28T04:42:29.159Z{"msg":"ds_connection connected from Ok(127.0.0.1:55992)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.622306345Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5692026-07-28T04:42:29.159Z{"msg":"ds_connection connected from Ok(127.0.0.1:50268)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.622321533Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5702026-07-28T04:42:29.159Z{"msg":"ds_connection connected from Ok(127.0.0.1:58334)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.622332672Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5712026-07-28T04:42:29.159Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.622803135Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"client":"1","":"downstairs","session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5722026-07-28T04:42:29.159Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.622820624Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"client":"0","":"downstairs","session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5732026-07-28T04:42:29.159Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.622871439Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"client":"2","":"downstairs","session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5742026-07-28T04:42:29.159Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.622986657Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5752026-07-28T04:42:29.159Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.622997626Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5762026-07-28T04:42:29.159Z{"msg":"[0]R flush_numbers: [5322, 5322, 5322, 5322, 5322]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.623020284Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"downstairs","":"downstairs","session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5772026-07-28T04:42:29.159Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.623032003Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"downstairs","":"downstairs","session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5782026-07-28T04:42:29.159Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.623043102Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"downstairs","":"downstairs","session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5792026-07-28T04:42:29.160Z{"msg":"[1]R flush_numbers: [5322, 5322, 5322, 5322, 5322]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.623054431Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"downstairs","":"downstairs","session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5802026-07-28T04:42:29.160Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.62306501Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"downstairs","":"downstairs","session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5812026-07-28T04:42:29.160Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.623076048Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"downstairs","":"downstairs","session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5822026-07-28T04:42:29.160Z{"msg":"[2]R flush_numbers: [5322, 5322, 5322, 5322, 5322]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.623087227Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"downstairs","":"downstairs","session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5832026-07-28T04:42:29.160Z{"msg":"[2]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.623098016Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"downstairs","":"downstairs","session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5842026-07-28T04:42:29.161Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.623108835Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"downstairs","":"downstairs","session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5852026-07-28T04:42:29.161Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.623119484Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"downstairs","":"downstairs","session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5862026-07-28T04:42:29.161Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.623130323Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"downstairs","":"downstairs","session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5872026-07-28T04:42:29.161Z{"msg":"Next flush: 5323","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.623140652Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"downstairs","":"downstairs","session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5882026-07-28T04:42:29.161Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.623151231Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"":"downstairs","":"downstairs","session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5892026-07-28T04:42:29.161Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.62316213Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5902026-07-28T04:42:29.161Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.623172019Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"client":"0","":"downstairs","session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5912026-07-28T04:42:29.161Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.623182678Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"client":"1","":"downstairs","session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5922026-07-28T04:42:29.161Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.623193227Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"client":"2","":"downstairs","session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5932026-07-28T04:42:29.161Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.623204006Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5942026-07-28T04:42:29.161Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.623213705Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5952026-07-28T04:42:29.162Z{"msg":"5ee40cce-78ea-4fa9-858d-00f9382f8e48 is now active with session: 0511d689-1997-4ca4-8410-2e41d94f8431","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.623224924Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5962026-07-28T04:42:29.163Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.623234393Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"session_id":"0511d689-1997-4ca4-8410-2e41d94f8431"}
5972026-07-28T04:42:29.163Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.623244732Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196}
5982026-07-28T04:42:29.163Z{"msg":"Activated sub_volume 5ee40cce-78ea-4fa9-858d-00f9382f8e48","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.623253251Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196}
5992026-07-28T04:42:29.163Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.62326263Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"task":"crutest"}
6002026-07-28T04:42:29.163ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
6012026-07-28T04:42:29.163ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
6022026-07-28T04:42:29.163ZRead and Verify all blocks (0..100 range:true)
6032026-07-28T04:42:29.320Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.67048548Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"task":"crutest"}
6042026-07-28T04:42:29.320Z{"msg":"Verify test completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.670499589Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"task":"crutest"}
6052026-07-28T04:42:29.320Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.670543985Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"task":"crutest"}
6062026-07-28T04:42:29.320Z{"msg":"CLIENT: Tests done. All submitted work has been ACK'd","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.670554844Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"task":"crutest"}
6072026-07-28T04:42:29.320Z----------------------------------------------------------------
6082026-07-28T04:42:29.320Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6092026-07-28T04:42:29.320ZStates: Active Active Active
6102026-07-28T04:42:29.320ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6112026-07-28T04:42:29.320Z 1000 Acked Read 50 Done Done Done false
6122026-07-28T04:42:29.320Z STATES DS:0 DS:1 DS:2 TOTAL
6132026-07-28T04:42:29.321Z Sent 0 0 0 0
6142026-07-28T04:42:29.321Z Done 1 1 1 3
6152026-07-28T04:42:29.321Z Skipped 0 0 0 0
6162026-07-28T04:42:29.321Z Error 0 0 0 0
6172026-07-28T04:42:29.321ZLast Flush: None None None
6182026-07-28T04:42:29.321ZDownstairs last five completed:
6192026-07-28T04:42:29.321ZUpstairs last five completed: 1000
6202026-07-28T04:42:29.321Z----------------------------------------------------------------
6212026-07-28T04:42:29.321Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6222026-07-28T04:42:29.321ZStates: Active Active Active
6232026-07-28T04:42:29.321ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6242026-07-28T04:42:29.321Z 1000 Acked Read 50 Done Done Done false
6252026-07-28T04:42:29.321Z STATES DS:0 DS:1 DS:2 TOTAL
6262026-07-28T04:42:29.321Z Sent 0 0 0 0
6272026-07-28T04:42:29.321Z Done 1 1 1 3
6282026-07-28T04:42:29.321Z Skipped 0 0 0 0
6292026-07-28T04:42:29.321Z Error 0 0 0 0
6302026-07-28T04:42:29.321ZLast Flush: None None None
6312026-07-28T04:42:29.321ZDownstairs last five completed:
6322026-07-28T04:42:29.321ZUpstairs last five completed: 1000
6332026-07-28T04:42:29.321Z{"msg":"CLIENT: Up:0 ds:2 act:6","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.670684411Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"task":"crutest"}
6342026-07-28T04:42:33.206Z----------------------------------------------------------------
6352026-07-28T04:42:33.206Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6362026-07-28T04:42:33.206ZDownstairs last five completed: 1001 1000
6372026-07-28T04:42:33.206ZUpstairs last five completed: 1001 1000
6382026-07-28T04:42:33.209Z----------------------------------------------------------------
6392026-07-28T04:42:33.209Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6402026-07-28T04:42:33.209ZDownstairs last five completed: 1001 1000
6412026-07-28T04:42:33.209ZUpstairs last five completed: 1001 1000
6422026-07-28T04:42:33.209Z{"msg":"CLIENT: Up:0 ds:0 act:6","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.671799655Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"task":"crutest"}
6432026-07-28T04:42:33.209Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.671820692Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196,"task":"crutest"}
6442026-07-28T04:42:33.209Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:31.671831551Z","hostname":"w-01KYKG6JQGQ8Z9NT39M0RRXVEB","pid":1196}
6452026-07-28T04:42:33.213ZRepair part 2 passed
6462026-07-28T04:42:33.213Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
6472026-07-28T04:42:33.225ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6482026-07-28T04:42:33.225Z 0 000-009 32 32 32 4002 4002 4002 F F F
6492026-07-28T04:42:33.228Z 1 010-019 32 32 32 4002 4002 4002 F F F
6502026-07-28T04:42:33.228Z 2 020-029 32 32 32 4002 4002 4002 F F F
6512026-07-28T04:42:33.228Z 3 030-039 32 32 32 4002 4002 4002 F F F
6522026-07-28T04:42:33.228Z 4 040-049 32 32 32 4002 4002 4002 F F F
6532026-07-28T04:42:33.228ZMax gen: 32, Max flush: 4002
6542026-07-28T04:42:33.228Zdump test passed
6552026-07-28T04:42:33.228Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
6562026-07-28T04:42:33.239Z Extent 1
6572026-07-28T04:42:33.242ZGEN 32 32 32
6582026-07-28T04:42:33.243ZFLUSH_ID 4002 4002 4002
6592026-07-28T04:42:33.243ZDIRTY
6602026-07-28T04:42:33.243Z
6612026-07-28T04:42:33.243ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
6622026-07-28T04:42:33.243Z010 A A A A A A
6632026-07-28T04:42:33.243Z011 A A A A A A
6642026-07-28T04:42:33.243Z012 A A A A A A
6652026-07-28T04:42:33.243Z013 A A A A A A
6662026-07-28T04:42:33.243Z014 A A A A A A
6672026-07-28T04:42:33.246Z015 A A A A A A
6682026-07-28T04:42:33.246Z016 A A A A A A
6692026-07-28T04:42:33.246Z017 A A A A A A
6702026-07-28T04:42:33.246Z018 A A A A A A
6712026-07-28T04:42:33.246Z019 A A A A A A
6722026-07-28T04:42:33.251Zdump extent test passed
6732026-07-28T04:42:33.251Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
6742026-07-28T04:42:33.258ZExtent 2 Block in extent 0 Actual block 20
6752026-07-28T04:42:33.258Z
6762026-07-28T04:42:33.261Z DATA SHA256 VER
6772026-07-28T04:42:33.261Z------ ---------------------------------------------------------------- ---
6782026-07-28T04:42:33.261Z 0 898496a78658d7a95ec4752a990bcdc3b1a5daacf94954e517b48ae9b8d2eacf A
6792026-07-28T04:42:33.262Z 1 898496a78658d7a95ec4752a990bcdc3b1a5daacf94954e517b48ae9b8d2eacf A
6802026-07-28T04:42:33.262Z 2 898496a78658d7a95ec4752a990bcdc3b1a5daacf94954e517b48ae9b8d2eacf A
6812026-07-28T04:42:33.262Z
6822026-07-28T04:42:33.262ZNONCES 0 1 2 DIFF
6832026-07-28T04:42:33.262Z------ ------------------------ ------------------------ ------------------------ -----
6842026-07-28T04:42:33.262Z 0 8ff2056a5a2325d5dfccffb9 8ff2056a5a2325d5dfccffb9 8ff2056a5a2325d5dfccffb9
6852026-07-28T04:42:33.262Z
6862026-07-28T04:42:33.262Z TAGS 0 1 2 DIFF
6872026-07-28T04:42:33.262Z------ -------------------------------- -------------------------------- -------------------------------- -----
6882026-07-28T04:42:33.262Z 0 0d9b2a5aa5c56398a44a80266a8c4b46 0d9b2a5aa5c56398a44a80266a8c4b46 0d9b2a5aa5c56398a44a80266a8c4b46
6892026-07-28T04:42:33.262Z
6902026-07-28T04:42:33.262ZHASHES 0 1 2 DIFF
6912026-07-28T04:42:33.264Z------ ---------------- ---------------- ---------------- -----
6922026-07-28T04:42:33.264Z
6932026-07-28T04:42:33.267Zdump block test passed
6942026-07-28T04:42:33.267ZInitial upstairs tests have completed, stopping all downstairs
6952026-07-28T04:42:38.284ZCreating 7 larger downstairs regions
6962026-07-28T04:42:38.491ZStarting 7 downstairs
6972026-07-28T04:42:43.496Zdsc restarted at PID: 1212
6982026-07-28T04:42:43.597ZNow do the replace-reconcile test
6992026-07-28T04:42:43.610ZUsing 8870 for the replacement port
7002026-07-28T04:46:35.109ZCompleted test: replace-reconcile
7012026-07-28T04:46:35.113ZNow do the replace-before-active test
7022026-07-28T04:50:27.218ZCompleted test: replace-before-active
7032026-07-28T04:50:27.221ZAll tests have completed, stopping all downstairs
7042026-07-28T04:50:27.234Z
7052026-07-28T04:50:27.237ZAll Tests have passed
7062026-07-28T04:50:27.237Z11:47 Test duration
7072026-07-28T04:50:27.237Z
7082026-07-28T04:50:27.237Zreal 11:47.373590543
7092026-07-28T04:50:27.237Zuser 16:18.162515419
7102026-07-28T04:50:27.237Zsys 11.792353161
7112026-07-28T04:50:27.237Ztrap 0.425278255
7122026-07-28T04:50:27.237Ztflt 0.044264248
7132026-07-28T04:50:27.237Zdflt 0.015090629
7142026-07-28T04:50:27.238Zkflt 0.001131301
7152026-07-28T04:50:27.238Zlock 15:47:21.295646132
7162026-07-28T04:50:27.238Zslp 50:01.626683009
7172026-07-28T04:50:27.238Zlat 1:45.796492676
7182026-07-28T04:50:27.238Zstop 0.024406071
7192026-07-28T04:50:27.238Z+ echo 'test-up-2region-encrypted ends'
7202026-07-28T04:50:27.238Ztest-up-2region-encrypted ends
7212026-07-28T04:50:32.240Zprocess exited: duration 717680 ms, exit code 0
7222026-07-28T04:50:32.241Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
7232026-07-28T04:51:32.270Zexec warning: : stdout descriptor may be held open by a background process; giving up!
7242026-07-28T04:51:32.270Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
7252026-07-28T04:51:32.281Zfound 17 output files
7262026-07-28T04:51:32.281Zuploading: /tmp/test_up-build/dsc-out.txt (59256 bytes)
7272026-07-28T04:51:33.294Zuploaded: /tmp/test_up-build/dsc-out.txt
7282026-07-28T04:51:33.294Zuploading: /tmp/test_up-build/test_up_out.txt (23079336 bytes)
7292026-07-28T04:51:34.446Zuploaded: /tmp/test_up-build/test_up_out.txt
7302026-07-28T04:51:34.448Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (550945 bytes)
7312026-07-28T04:51:35.463Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
7322026-07-28T04:51:35.465Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (548323 bytes)
7332026-07-28T04:51:36.481Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
7342026-07-28T04:51:36.481Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (978666 bytes)
7352026-07-28T04:51:37.500Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
7362026-07-28T04:51:37.503Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (543060 bytes)
7372026-07-28T04:51:38.516Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
7382026-07-28T04:51:38.516Zuploading: /tmp/test_up-build/dsc/downstairs-8850.txt (1343656 bytes)
7392026-07-28T04:51:39.539Zuploaded: /tmp/test_up-build/dsc/downstairs-8850.txt
7402026-07-28T04:51:39.542Zuploading: /tmp/test_up-build/dsc/downstairs-8860.txt (47330 bytes)
7412026-07-28T04:51:40.551Zuploaded: /tmp/test_up-build/dsc/downstairs-8860.txt
7422026-07-28T04:51:40.551Zuploading: /tmp/test_up-build/dsc/downstairs-8870.txt (3613264 bytes)
7432026-07-28T04:51:41.585Zuploaded: /tmp/test_up-build/dsc/downstairs-8870.txt
7442026-07-28T04:51:41.585Zuploading: /tmp/debug/df.txt (1270 bytes)
7452026-07-28T04:51:42.595Zuploaded: /tmp/debug/df.txt
7462026-07-28T04:51:42.598Zuploading: /tmp/debug/dtrace.txt (1019366 bytes)
7472026-07-28T04:51:42.611Zupload warning: file "/tmp/debug/dtrace.txt" changed size mid upload: 1019366 -> 1156259
7482026-07-28T04:51:43.618Zuploaded: /tmp/debug/dtrace.txt
7492026-07-28T04:51:43.621Zuploading: /tmp/debug/iostat.txt (229855 bytes)
7502026-07-28T04:51:43.628Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 229855 -> 233122
7512026-07-28T04:51:44.636Zuploaded: /tmp/debug/iostat.txt
7522026-07-28T04:51:44.639Zuploading: /tmp/debug/mpstat.txt (561390 bytes)
7532026-07-28T04:51:44.648Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 561390 -> 570102
7542026-07-28T04:51:45.656Zuploaded: /tmp/debug/mpstat.txt
7552026-07-28T04:51:45.659Zuploading: /tmp/debug/paging.txt (98900 bytes)
7562026-07-28T04:51:45.665Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 98900 -> 100514
7572026-07-28T04:51:46.672Zuploaded: /tmp/debug/paging.txt
7582026-07-28T04:51:46.672Zuploading: /tmp/debug/prstat.txt (1076466 bytes)
7592026-07-28T04:51:46.690Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 1076466 -> 1094612
7602026-07-28T04:51:47.696Zuploaded: /tmp/debug/prstat.txt
7612026-07-28T04:51:47.699Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
7622026-07-28T04:51:48.706Zuploaded: /tmp/debug/psrinfo.txt
7632026-07-28T04:51:48.708Zuploading: /tmp/debug/upstairs-info.txt (166243 bytes)
7642026-07-28T04:51:49.720Zuploaded: /tmp/debug/upstairs-info.txt