01KRZ99JJE5ABSWTGTVQY0MP88: test-up-2region-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01KRZ9A3YXBVMQ33JDSJ7ZE82A

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12026-05-19T05:06:43.032Zjob dependencies complete; ready to run (waiting for 13 m 6 s)
22026-05-19T05:07:42.380Zjob assigned to worker 01KRZA2C4BS76SB2900SK3KRE4 [factory aws, i-0931c42a82ddd1395] (queued for 59 s)
32026-05-19T05:07:49.197Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42026-05-19T05:07:51.185Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52026-05-19T05:07:51.189Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62026-05-19T05:07:53.111Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72026-05-19T05:07:53.124Zdownloading input: /input/build/work/bins/crutest.gz
82026-05-19T05:07:56.001Zdownloaded input: /input/build/work/bins/crutest.gz
92026-05-19T05:07:56.007Zdownloading input: /input/build/work/bins/dsc.gz
102026-05-19T05:07:56.830Zdownloaded input: /input/build/work/bins/dsc.gz
112026-05-19T05:07:56.835Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122026-05-19T05:07:56.864Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132026-05-19T05:07:56.864Zdownloading input: /input/build/work/scripts/test_ds.sh
142026-05-19T05:07:56.893Zdownloaded input: /input/build/work/scripts/test_ds.sh
152026-05-19T05:07:56.893Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162026-05-19T05:07:56.924Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172026-05-19T05:07:56.924Zdownloading input: /input/build/work/scripts/test_repair.sh
182026-05-19T05:07:56.953Zdownloaded input: /input/build/work/scripts/test_repair.sh
192026-05-19T05:07:56.953Zdownloading input: /input/build/work/scripts/test_replay.sh
202026-05-19T05:07:56.983Zdownloaded input: /input/build/work/scripts/test_replay.sh
212026-05-19T05:07:56.983Zdownloading input: /input/build/work/scripts/test_up.sh
222026-05-19T05:07:57.013Zdownloaded input: /input/build/work/scripts/test_up.sh
232026-05-19T05:07:57.017Zdownloading input: /input/build/work/scripts/upstairs_info.d
242026-05-19T05:07:57.041Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252026-05-19T05:07:57.045Zdownloading input: /input/build/tmp/cargo-test-out.log
262026-05-19T05:07:57.574Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272026-05-19T05:07:57.574Zstarting task 0: "setup"
282026-05-19T05:07:57.580Z++ uname -s
292026-05-19T05:07:57.583Z+ kern=SunOS
302026-05-19T05:07:57.584Z+ build_user=build
312026-05-19T05:07:57.584Z+ build_uid=12345
322026-05-19T05:07:57.584Z+ work_dir=/work
332026-05-19T05:07:57.584Z+ input_dir=/input
342026-05-19T05:07:57.584Z+ [[ 0 == 12345 ]]
352026-05-19T05:07:57.584Z+ case "$kern" in
362026-05-19T05:07:57.584Z+ groupadd -g 12345 build
372026-05-19T05:07:57.584Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382026-05-19T05:07:59.590Z+ zfs create -o mountpoint=/work rpool/work
392026-05-19T05:07:59.854Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402026-05-19T05:07:59.857Z+ home_fs=zfs
412026-05-19T05:07:59.858Z+ [[ zfs == autofs ]]
422026-05-19T05:07:59.858Z+ mkdir -p /home/build
432026-05-19T05:07:59.858Z+ chown build:build /home/build /work
442026-05-19T05:08:00.859Z+ chmod 0700 /home/build /work
452026-05-19T05:08:00.862Zprocess exited: duration 3285 ms, exit code 0
 
462026-05-19T05:08:00.869Zstarting task 1: "authentication"
472026-05-19T05:08:00.887Zprocess exited: duration 17 ms, exit code 0
 
482026-05-19T05:08:00.894Zstarting task 2: "build"
492026-05-19T05:08:00.897Z+ banner cores
502026-05-19T05:08:00.901Z
512026-05-19T05:08:00.901Z #### #### ##### ###### ####
522026-05-19T05:08:00.901Z # # # # # # # #
532026-05-19T05:08:00.901Z # # # # # ##### ####
542026-05-19T05:08:00.901Z # # # ##### # #
552026-05-19T05:08:00.901Z # # # # # # # # #
562026-05-19T05:08:00.901Z #### #### # # ###### ####
572026-05-19T05:08:00.901Z
582026-05-19T05:08:00.901Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
592026-05-19T05:08:00.905Z+ banner unpack
602026-05-19T05:08:00.908Z
612026-05-19T05:08:00.908Z # # # # ##### ## #### # #
622026-05-19T05:08:00.908Z # # ## # # # # # # # # #
632026-05-19T05:08:00.908Z # # # # # # # # # # ####
642026-05-19T05:08:00.909Z # # # # # ##### ###### # # #
652026-05-19T05:08:00.909Z # # # ## # # # # # # #
662026-05-19T05:08:00.909Z #### # # # # # #### # #
672026-05-19T05:08:00.909Z
682026-05-19T05:08:00.909Z+ mkdir -p /var/tmp/bins
692026-05-19T05:08:00.909Z+ for t in "$input/bins/"*.gz
702026-05-19T05:08:00.912Z++ basename /input/build/work/bins/crucible-downstairs.gz
712026-05-19T05:08:00.912Z+ b=crucible-downstairs.gz
722026-05-19T05:08:00.912Z+ b=crucible-downstairs
732026-05-19T05:08:00.912Z+ gunzip
742026-05-19T05:08:03.583Z+ chmod +x /var/tmp/bins/crucible-downstairs
752026-05-19T05:08:03.586Z+ for t in "$input/bins/"*.gz
762026-05-19T05:08:03.587Z++ basename /input/build/work/bins/crucible-hammer.gz
772026-05-19T05:08:03.590Z+ b=crucible-hammer.gz
782026-05-19T05:08:03.590Z+ b=crucible-hammer
792026-05-19T05:08:03.590Z+ gunzip
802026-05-19T05:08:06.087Z+ chmod +x /var/tmp/bins/crucible-hammer
812026-05-19T05:08:06.090Z+ for t in "$input/bins/"*.gz
822026-05-19T05:08:06.093Z++ basename /input/build/work/bins/crutest.gz
832026-05-19T05:08:06.093Z+ b=crutest.gz
842026-05-19T05:08:06.093Z+ b=crutest
852026-05-19T05:08:06.093Z+ gunzip
862026-05-19T05:08:08.728Z+ chmod +x /var/tmp/bins/crutest
872026-05-19T05:08:08.732Z+ for t in "$input/bins/"*.gz
882026-05-19T05:08:08.732Z++ basename /input/build/work/bins/dsc.gz
892026-05-19T05:08:08.735Z+ b=dsc.gz
902026-05-19T05:08:08.735Z+ b=dsc
912026-05-19T05:08:08.735Z+ gunzip
922026-05-19T05:08:09.932Z+ chmod +x /var/tmp/bins/dsc
932026-05-19T05:08:09.935Z+ export BINDIR=/var/tmp/bins
942026-05-19T05:08:09.935Z+ BINDIR=/var/tmp/bins
952026-05-19T05:08:09.935Z+ jobpid=1091
962026-05-19T05:08:09.935Z+ echo 'Setup debug logging'
972026-05-19T05:08:09.935Z+ mkdir /tmp/debug
982026-05-19T05:08:09.935ZSetup debug logging
992026-05-19T05:08:09.935Z+ sleep 3600
1002026-05-19T05:08:09.939Z+ psrinfo -v
1012026-05-19T05:08:09.942Z+ df -h
1022026-05-19T05:08:09.945Z+ prstat -d d -mLc 1
1032026-05-19T05:08:09.945Z+ iostat -T d -xn 1
1042026-05-19T05:08:09.947Z+ mpstat -T d 1
1052026-05-19T05:08:09.947Z+ vmstat -T d -p 1
1062026-05-19T05:08:09.947Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1072026-05-19T05:08:09.948Z+ banner test_up_2r_encrypted
1082026-05-19T05:08:09.948Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1092026-05-19T05:08:09.958Z #####
1102026-05-19T05:08:09.958Z ##### ###### #### ##### # # ##### # # #####
1112026-05-19T05:08:09.959Z # # # # # # # # # # #
1122026-05-19T05:08:09.959Z # ##### #### # # # # # ##### # #
1132026-05-19T05:08:09.959Z # # # # # # ##### # #####
1142026-05-19T05:08:09.959Z # # # # # # # # # # #
1152026-05-19T05:08:09.959Z # ###### #### # ####### #### # ####### ####### # #
1162026-05-19T05:08:09.959Z
1172026-05-19T05:08:09.962Z+ ptime -m bash /input/build/work/scripts/test_up.sh -r 2 -N encrypted
1182026-05-19T05:08:09.967Z/input/build/work
1192026-05-19T05:08:09.975ZUsing 2 region sets
1202026-05-19T05:08:09.975ZTurn off color for downstairs dump
1212026-05-19T05:08:10.010ZUpstairs using key: ZWnKbMVewk/OQ6RT+cwMAypB2RTj2mH6fJ+xiu6b8Eg=
1222026-05-19T05:08:10.013Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1232026-05-19T05:08:10.013ZCreating 6 downstairs regions
1242026-05-19T05:08:10.238ZStarting 6 downstairs
1252026-05-19T05:08:10.241Zdsc started at PID: 1139
1262026-05-19T05:08:15.330ZDisable automatic restart on all downstairs
1272026-05-19T05:08:15.342Z
1282026-05-19T05:08:15.345ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1292026-05-19T05:08:15.345ZRunning test: span
1302026-05-19T05:08:19.449ZCompleted test: span
1312026-05-19T05:08:24.456ZRunning test: big
1322026-05-19T05:08:28.756ZCompleted test: big
1332026-05-19T05:08:33.764ZRunning test: dep
1342026-05-19T05:08:53.111ZCompleted test: dep
1352026-05-19T05:08:58.117ZRunning test: balloon
1362026-05-19T05:11:22.559ZCompleted test: balloon
1372026-05-19T05:11:27.566ZRunning test: deactivate
1382026-05-19T05:11:31.912ZCompleted test: deactivate
1392026-05-19T05:11:36.919ZRunning hammer
1402026-05-19T05:11:44.144ZRun repair tests
1412026-05-19T05:11:44.148Z/var/tmp/bins/crutest fill -g 31 -q --key ZWnKbMVewk/OQ6RT+cwMAypB2RTj2mH6fJ+xiu6b8Eg= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1422026-05-19T05:11:44.166Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.499322572Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"task":"crutest"}
1432026-05-19T05:11:44.171Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.499516333Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"task":"crutest"}
1442026-05-19T05:11:44.171Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.500119536Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"task":"crutest"}
1452026-05-19T05:11:44.171Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.500366522Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182}
1462026-05-19T05:11:44.171Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c9d557ea5b0ac329d65f72d3a87737e706c99cd3\",\n git_commit_timestamp: \"2026-05-19T04:52:54.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.500418117Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182}
1472026-05-19T05:11:44.171Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.500427986Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182}
1482026-05-19T05:11:44.171Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.500440155Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182}
1492026-05-19T05:11:44.171Z{"msg":"Crucible c5b2bbd2-8910-4cea-9133-6ee20745e4eb has session id: 37e83cc4-a643-4036-bf77-4bf0623a4fd9","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.500752486Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1502026-05-19T05:11:44.171Z{"msg":"Upstairs opts: Upstairs UUID: c5b2bbd2-8910-4cea-9133-6ee20745e4eb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.500791742Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1512026-05-19T05:11:44.171Z{"msg":"Crucible stats registered with UUID: c5b2bbd2-8910-4cea-9133-6ee20745e4eb","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.500803551Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1522026-05-19T05:11:44.172Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-19T05:11:42.50081355Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1532026-05-19T05:11:44.172Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.500937778Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1542026-05-19T05:11:44.172Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.500960116Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1552026-05-19T05:11:44.172Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.500970595Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1562026-05-19T05:11:44.172Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.50185532Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"task":"crutest"}
1572026-05-19T05:11:44.172Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.501912575Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182}
1582026-05-19T05:11:44.172Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c9d557ea5b0ac329d65f72d3a87737e706c99cd3\",\n git_commit_timestamp: \"2026-05-19T04:52:54.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.501937232Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182}
1592026-05-19T05:11:44.172Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.501950231Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182}
1602026-05-19T05:11:44.172Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.50196181Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182}
1612026-05-19T05:11:44.172Z{"msg":"Crucible c5b2bbd2-8910-4cea-9133-6ee20745e4eb has session id: 846c3a55-57db-401d-ada8-1f544b5535b8","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.502143393Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
1622026-05-19T05:11:44.172Z{"msg":"Upstairs opts: Upstairs UUID: c5b2bbd2-8910-4cea-9133-6ee20745e4eb, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.502158801Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
1632026-05-19T05:11:44.172Z{"msg":"Crucible stats registered with UUID: c5b2bbd2-8910-4cea-9133-6ee20745e4eb","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.50216912Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
1642026-05-19T05:11:44.172Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-19T05:11:42.502182619Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
1652026-05-19T05:11:44.172Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.502191518Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
1662026-05-19T05:11:44.172Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.502202037Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
1672026-05-19T05:11:44.172Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.502213046Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
1682026-05-19T05:11:44.172Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.502368871Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"task":"crutest"}
1692026-05-19T05:11:44.172Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.50238156Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182}
1702026-05-19T05:11:44.172Z{"msg":"c5b2bbd2-8910-4cea-9133-6ee20745e4eb active request set","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.502415567Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1712026-05-19T05:11:44.172Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.502426326Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"client":"0","":"downstairs","session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1722026-05-19T05:11:44.172Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.502444124Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"client":"1","":"downstairs","session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1732026-05-19T05:11:44.172Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.502454363Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"client":"2","":"downstairs","session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1742026-05-19T05:11:44.172Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.502464322Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1752026-05-19T05:11:44.172Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.502474701Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1762026-05-19T05:11:44.172Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.50248508Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1772026-05-19T05:11:44.172Z{"msg":"ds_connection connected from Ok(127.0.0.1:54490)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.502514477Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1782026-05-19T05:11:44.172Z{"msg":"ds_connection connected from Ok(127.0.0.1:57221)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.502526016Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1792026-05-19T05:11:44.172Z{"msg":"ds_connection connected from Ok(127.0.0.1:46473)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.502536845Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1802026-05-19T05:11:44.172Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503232139Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"client":"1","":"downstairs","session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1812026-05-19T05:11:44.172Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503267276Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"client":"2","":"downstairs","session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1822026-05-19T05:11:44.173Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503279704Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"client":"0","":"downstairs","session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1832026-05-19T05:11:44.173Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503421491Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1842026-05-19T05:11:44.173Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503473096Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1852026-05-19T05:11:44.177Z{"msg":"[0]R flush_numbers: [3997, 3997, 3997, 3997, 3997]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503510752Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"downstairs","":"downstairs","session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1862026-05-19T05:11:44.177Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503521871Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"downstairs","":"downstairs","session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1872026-05-19T05:11:44.177Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.50353262Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"downstairs","":"downstairs","session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1882026-05-19T05:11:44.177Z{"msg":"[1]R flush_numbers: [3997, 3997, 3997, 3997, 3997]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503543549Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"downstairs","":"downstairs","session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1892026-05-19T05:11:44.177Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503553968Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"downstairs","":"downstairs","session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1902026-05-19T05:11:44.177Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503564477Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"downstairs","":"downstairs","session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1912026-05-19T05:11:44.177Z{"msg":"[2]R flush_numbers: [3997, 3997, 3997, 3997, 3997]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503575126Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"downstairs","":"downstairs","session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1922026-05-19T05:11:44.177Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503585245Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"downstairs","":"downstairs","session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1932026-05-19T05:11:44.177Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503595954Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"downstairs","":"downstairs","session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1942026-05-19T05:11:44.177Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503606183Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"downstairs","":"downstairs","session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1952026-05-19T05:11:44.177Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503616662Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"downstairs","":"downstairs","session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1962026-05-19T05:11:44.177Z{"msg":"Next flush: 3998","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503626811Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"downstairs","":"downstairs","session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1972026-05-19T05:11:44.177Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.50363706Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"downstairs","":"downstairs","session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1982026-05-19T05:11:44.177Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503647769Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
1992026-05-19T05:11:44.177Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503657468Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"client":"0","":"downstairs","session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
2002026-05-19T05:11:44.177Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503667707Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"client":"1","":"downstairs","session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
2012026-05-19T05:11:44.177Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503677876Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"client":"2","":"downstairs","session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
2022026-05-19T05:11:44.177Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503687675Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
2032026-05-19T05:11:44.177Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503697664Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
2042026-05-19T05:11:44.177Z{"msg":"c5b2bbd2-8910-4cea-9133-6ee20745e4eb is now active with session: 37e83cc4-a643-4036-bf77-4bf0623a4fd9","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503707993Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
2052026-05-19T05:11:44.177Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503716763Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"session_id":"37e83cc4-a643-4036-bf77-4bf0623a4fd9"}
2062026-05-19T05:11:44.177Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503726402Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182}
2072026-05-19T05:11:44.177Z{"msg":"Activated sub_volume c5b2bbd2-8910-4cea-9133-6ee20745e4eb","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503734781Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182}
2082026-05-19T05:11:44.177Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.50374297Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182}
2092026-05-19T05:11:44.177Z{"msg":"c5b2bbd2-8910-4cea-9133-6ee20745e4eb active request set","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503751279Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2102026-05-19T05:11:44.177Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503764408Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"client":"0","":"downstairs","session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2112026-05-19T05:11:44.177Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503774437Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"client":"1","":"downstairs","session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2122026-05-19T05:11:44.177Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503784596Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"client":"2","":"downstairs","session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2132026-05-19T05:11:44.177Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503794195Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2142026-05-19T05:11:44.177Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503804214Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2152026-05-19T05:11:44.177Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503814283Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2162026-05-19T05:11:44.177Z{"msg":"ds_connection connected from Ok(127.0.0.1:48697)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503857919Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2172026-05-19T05:11:44.178Z{"msg":"ds_connection connected from Ok(127.0.0.1:44455)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503868368Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2182026-05-19T05:11:44.178Z{"msg":"ds_connection connected from Ok(127.0.0.1:62028)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.503882787Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2192026-05-19T05:11:44.178Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.504113495Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"client":"2","":"downstairs","session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2202026-05-19T05:11:44.178Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.504206736Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"client":"0","":"downstairs","session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2212026-05-19T05:11:44.178Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.504239513Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"client":"1","":"downstairs","session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2222026-05-19T05:11:44.178Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.504250882Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2232026-05-19T05:11:44.178Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.504389558Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2242026-05-19T05:11:44.178Z{"msg":"[0]R flush_numbers: [5321, 5318, 5320, 5320, 5310]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.504413316Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"downstairs","":"downstairs","session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2252026-05-19T05:11:44.178Z{"msg":"[0]R generation: [14, 13, 14, 14, 11]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.504425165Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"downstairs","":"downstairs","session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2262026-05-19T05:11:44.178Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.504436074Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"downstairs","":"downstairs","session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2272026-05-19T05:11:44.178Z{"msg":"[1]R flush_numbers: [5321, 5318, 5320, 5320, 5310]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.504452232Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"downstairs","":"downstairs","session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2282026-05-19T05:11:44.178Z{"msg":"[1]R generation: [14, 13, 14, 14, 11]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.504462941Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"downstairs","":"downstairs","session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2292026-05-19T05:11:44.178Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.504497658Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"downstairs","":"downstairs","session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2302026-05-19T05:11:44.178Z{"msg":"[2]R flush_numbers: [5321, 5318, 5320, 5320, 5310]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.504543974Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"downstairs","":"downstairs","session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2312026-05-19T05:11:44.178Z{"msg":"[2]R generation: [14, 13, 14, 14, 11]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.504561692Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"downstairs","":"downstairs","session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2322026-05-19T05:11:44.178Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.50457951Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"downstairs","":"downstairs","session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2332026-05-19T05:11:44.178Z{"msg":"Max found gen is 15","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.504589449Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"downstairs","":"downstairs","session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2342026-05-19T05:11:44.179Z{"msg":"Generation requested: 31 >= found:15","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.504600018Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"downstairs","":"downstairs","session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2352026-05-19T05:11:44.179Z{"msg":"Next flush: 5322","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.504610017Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"downstairs","":"downstairs","session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2362026-05-19T05:11:44.179Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.504619886Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"":"downstairs","":"downstairs","session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2372026-05-19T05:11:44.179Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.504630155Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2382026-05-19T05:11:44.179Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.504643124Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"client":"0","":"downstairs","session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2392026-05-19T05:11:44.179Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.504653013Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"client":"1","":"downstairs","session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2402026-05-19T05:11:44.179Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.504663352Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"client":"2","":"downstairs","session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2412026-05-19T05:11:44.179Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.504673221Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2422026-05-19T05:11:44.179Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.50468294Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2432026-05-19T05:11:44.179Z{"msg":"c5b2bbd2-8910-4cea-9133-6ee20745e4eb is now active with session: 846c3a55-57db-401d-ada8-1f544b5535b8","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.504693549Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2442026-05-19T05:11:44.179Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.504702789Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"session_id":"846c3a55-57db-401d-ada8-1f544b5535b8"}
2452026-05-19T05:11:44.179Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.504712608Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182}
2462026-05-19T05:11:44.179Z{"msg":"Activated sub_volume c5b2bbd2-8910-4cea-9133-6ee20745e4eb","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.504721467Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182}
2472026-05-19T05:11:44.179Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","Disk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
2482026-05-19T05:11:44.179Zlevel":30,"time":"2026-05-19T05:11:42.504735315Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"task":"crutest"}
2492026-05-19T05:11:44.179Z{"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.504763653Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"task":"crutest"}
2502026-05-19T05:11:44.196ZRead and Verify all blocks (0..100 range:false)
2512026-05-19T05:11:44.243Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30----------------------------------------------------------------
2522026-05-19T05:11:44.244Z,"time":"2026-05-19T05:11:42.577033966Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4" Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2532026-05-19T05:11:44.244Z,States:"pid":1182 Active," Activetask" Active:
2542026-05-19T05:11:44.244Z"JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2552026-05-19T05:11:44.244Zcrutest"}
2562026-05-19T05:11:44.244Z 1002 Acked Read 50{ Done" Donemsg" Done:" false
2572026-05-19T05:11:44.244ZCLIENT: Tests done. All submitted work has been ACK'd","v":0,"name":" STATES DS:0 DS:1 DS:2 TOTAL
2582026-05-19T05:11:44.244Zcrucible Sent "," 0 level 0 " 0 : 0
2592026-05-19T05:11:44.244Z30 Done 1 1 1 3
2602026-05-19T05:11:44.244Z Skipped 0 0 0 0
2612026-05-19T05:11:44.244Z Error 0 0 0 0
2622026-05-19T05:11:44.247Z,"timeLast Flush: ":"Some(JobId(1001)) 2026-05-19T05:11:42.577070832ZSome(JobId(1001)) "Some(JobId(1001)) ,
2632026-05-19T05:11:44.247Z"Downstairs last five completed:hostname":" 1001 1000w-01KRZA2C4BS76SB2900SK3KRE4"
2642026-05-19T05:11:44.247Z,"pid":1182Upstairs last five completed: ," 1002task" 1001: 1000"
2652026-05-19T05:11:44.247Zcrutest"}
2662026-05-19T05:11:44.247Z----------------------------------------------------------------
2672026-05-19T05:11:44.247Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2682026-05-19T05:11:44.247ZStates: Active Active Active
2692026-05-19T05:11:44.247ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2702026-05-19T05:11:44.247Z 1002 Acked Read 50 Done Done Done false
2712026-05-19T05:11:44.247Z STATES DS:0 DS:1 DS:2 TOTAL
2722026-05-19T05:11:44.247Z Sent 0 0 0 0
2732026-05-19T05:11:44.247Z Done 1 1 1 3
2742026-05-19T05:11:44.247Z Skipped 0 0 0 0
2752026-05-19T05:11:44.247Z Error 0 0 0 0
2762026-05-19T05:11:44.247ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2772026-05-19T05:11:44.247ZDownstairs last five completed: 1001 1000
2782026-05-19T05:11:44.247ZUpstairs last five completed: 1002 1001 1000
2792026-05-19T05:11:44.247Z{"msg":"CLIENT: Up:0 ds:2 act:6","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:42.577141135Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"task":"crutest"}
2802026-05-19T05:11:48.245Z----------------------------------------------------------------
2812026-05-19T05:11:48.245Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2822026-05-19T05:11:48.245ZDownstairs last five completed: 1003 1002 1001 1000
2832026-05-19T05:11:48.245ZUpstairs last five completed: 1003 1002 1001 1000
2842026-05-19T05:11:48.249Z----------------------------------------------------------------
2852026-05-19T05:11:48.249Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2862026-05-19T05:11:48.249ZDownstairs last five completed: 1003 1002 1001 1000
2872026-05-19T05:11:48.249ZUpstairs last five completed: 1003 1002 1001 1000
2882026-05-19T05:11:48.249Z{"msg":"CLIENT: Up:0 ds:0 act:6","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.578261269Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"task":"crutest"}
2892026-05-19T05:11:48.249Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.578327972Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1182,"task":"crutest"}
2902026-05-19T05:11:48.254ZRepair setup passed
2912026-05-19T05:11:48.256ZCopy the region for /var/tmp/test_up-build/8830
2922026-05-19T05:11:48.259Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2932026-05-19T05:11:48.259Z/var/tmp/bins/crutest fill -g 32 -q --key ZWnKbMVewk/OQ6RT+cwMAypB2RTj2mH6fJ+xiu6b8Eg= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2942026-05-19T05:11:48.278Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.610777305Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"task":"crutest"}
2952026-05-19T05:11:48.281Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.611124492Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"task":"crutest"}
2962026-05-19T05:11:48.281Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.6118849Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"task":"crutest"}
2972026-05-19T05:11:48.281Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.612140165Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188}
2982026-05-19T05:11:48.281Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c9d557ea5b0ac329d65f72d3a87737e706c99cd3\",\n git_commit_timestamp: \"2026-05-19T04:52:54.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.612185151Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188}
2992026-05-19T05:11:48.281Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.61219866Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188}
3002026-05-19T05:11:48.281Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.612211039Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188}
3012026-05-19T05:11:48.281Z{"msg":"Crucible 6a0c33d0-dd78-4225-83d9-ff4b72a7718b has session id: 883fe234-8c20-4994-a6b0-c112c510446b","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.61261716Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3022026-05-19T05:11:48.281Z{"msg":"Upstairs opts: Upstairs UUID: 6a0c33d0-dd78-4225-83d9-ff4b72a7718b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.612644377Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3032026-05-19T05:11:48.281Z{"msg":"Crucible stats registered with UUID: 6a0c33d0-dd78-4225-83d9-ff4b72a7718b","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.612654796Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3042026-05-19T05:11:48.281Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-19T05:11:46.612664695Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3052026-05-19T05:11:48.281Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.612800112Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3062026-05-19T05:11:48.281Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.612843188Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3072026-05-19T05:11:48.281Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.612855117Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3082026-05-19T05:11:48.281Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.613696977Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"task":"crutest"}
3092026-05-19T05:11:48.281Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.613751282Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188}
3102026-05-19T05:11:48.281Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c9d557ea5b0ac329d65f72d3a87737e706c99cd3\",\n git_commit_timestamp: \"2026-05-19T04:52:54.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.613775929Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188}
3112026-05-19T05:11:48.281Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.613788818Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188}
3122026-05-19T05:11:48.281Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.613800777Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188}
3132026-05-19T05:11:48.281Z{"msg":"Crucible 6a0c33d0-dd78-4225-83d9-ff4b72a7718b has session id: bcd96321-2e2c-499e-98e3-2688765c8f5f","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.614032965Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3142026-05-19T05:11:48.281Z{"msg":"Upstairs opts: Upstairs UUID: 6a0c33d0-dd78-4225-83d9-ff4b72a7718b, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.6140809Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3152026-05-19T05:11:48.281Z{"msg":"Crucible stats registered with UUID: 6a0c33d0-dd78-4225-83d9-ff4b72a7718b","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.614095199Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3162026-05-19T05:11:48.281Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-19T05:11:46.614104268Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3172026-05-19T05:11:48.281Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.614113247Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3182026-05-19T05:11:48.281Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.614124296Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3192026-05-19T05:11:48.285Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.614135025Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3202026-05-19T05:11:48.285Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.614215367Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"task":"crutest"}
3212026-05-19T05:11:48.285Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.614232716Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188}
3222026-05-19T05:11:48.285Z{"msg":"6a0c33d0-dd78-4225-83d9-ff4b72a7718b active request set","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.614261263Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3232026-05-19T05:11:48.285Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.61429068Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"client":"0","":"downstairs","session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3242026-05-19T05:11:48.285Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.614302259Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"client":"1","":"downstairs","session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3252026-05-19T05:11:48.285Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.614316698Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"client":"2","":"downstairs","session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3262026-05-19T05:11:48.285Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.614327067Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3272026-05-19T05:11:48.285Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.614341995Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3282026-05-19T05:11:48.286Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.614356564Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3292026-05-19T05:11:48.286Z{"msg":"ds_connection connected from Ok(127.0.0.1:36597)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.614367723Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3302026-05-19T05:11:48.286Z{"msg":"ds_connection connected from Ok(127.0.0.1:53867)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.614378722Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3312026-05-19T05:11:48.286Z{"msg":"ds_connection connected from Ok(127.0.0.1:56612)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.614390191Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3322026-05-19T05:11:48.286Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.61512611Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"client":"2","":"downstairs","session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3332026-05-19T05:11:48.286Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615169056Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"client":"1","":"downstairs","session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3342026-05-19T05:11:48.286Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615186115Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"client":"0","":"downstairs","session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3352026-05-19T05:11:48.286Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615294264Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3362026-05-19T05:11:48.286Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615320492Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3372026-05-19T05:11:48.286Z{"msg":"[0]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615371557Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"downstairs","":"downstairs","session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3382026-05-19T05:11:48.286Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615386706Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"downstairs","":"downstairs","session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3392026-05-19T05:11:48.286Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615396815Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"downstairs","":"downstairs","session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3402026-05-19T05:11:48.286Z{"msg":"[1]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615411093Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"downstairs","":"downstairs","session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3412026-05-19T05:11:48.286Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615420832Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"downstairs","":"downstairs","session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3422026-05-19T05:11:48.286Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615435061Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"downstairs","":"downstairs","session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3432026-05-19T05:11:48.286Z{"msg":"[2]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.61544517Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"downstairs","":"downstairs","session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3442026-05-19T05:11:48.286Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615459369Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"downstairs","":"downstairs","session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3452026-05-19T05:11:48.286Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615469338Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"downstairs","":"downstairs","session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3462026-05-19T05:11:48.286Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615483016Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"downstairs","":"downstairs","session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3472026-05-19T05:11:48.286Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615492885Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"downstairs","":"downstairs","session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3482026-05-19T05:11:48.286Z{"msg":"Next flush: 3999","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615506304Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"downstairs","":"downstairs","session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3492026-05-19T05:11:48.286Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615516263Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"downstairs","":"downstairs","session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3502026-05-19T05:11:48.286Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615526182Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3512026-05-19T05:11:48.286Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615539711Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"client":"0","":"downstairs","session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3522026-05-19T05:11:48.286Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.61554958Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"client":"1","":"downstairs","session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3532026-05-19T05:11:48.286Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615566898Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"client":"2","":"downstairs","session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3542026-05-19T05:11:48.286Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615576687Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3552026-05-19T05:11:48.286Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615590576Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3562026-05-19T05:11:48.286Z{"msg":"6a0c33d0-dd78-4225-83d9-ff4b72a7718b is now active with session: 883fe234-8c20-4994-a6b0-c112c510446b","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615601235Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3572026-05-19T05:11:48.286Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615614064Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"session_id":"883fe234-8c20-4994-a6b0-c112c510446b"}
3582026-05-19T05:11:48.286Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615623263Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188}
3592026-05-19T05:11:48.286Z{"msg":"Activated sub_volume 6a0c33d0-dd78-4225-83d9-ff4b72a7718b","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615635692Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188}
3602026-05-19T05:11:48.286Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615643891Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188}
3612026-05-19T05:11:48.286Z{"msg":"6a0c33d0-dd78-4225-83d9-ff4b72a7718b active request set","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.61565226Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3622026-05-19T05:11:48.286Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615669019Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"client":"0","":"downstairs","session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3632026-05-19T05:11:48.287Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615678628Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"client":"1","":"downstairs","session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3642026-05-19T05:11:48.287Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615688487Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"client":"2","":"downstairs","session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3652026-05-19T05:11:48.287Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615698606Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3662026-05-19T05:11:48.287Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615708875Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3672026-05-19T05:11:48.287Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615718904Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3682026-05-19T05:11:48.287Z{"msg":"ds_connection connected from Ok(127.0.0.1:57660)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615733252Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3692026-05-19T05:11:48.287Z{"msg":"ds_connection connected from Ok(127.0.0.1:42202)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615776728Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3702026-05-19T05:11:48.287Z{"msg":"ds_connection connected from Ok(127.0.0.1:46557)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.615821924Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3712026-05-19T05:11:48.287Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.616143233Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"client":"0","":"downstairs","session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3722026-05-19T05:11:48.287Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.61617701Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"client":"2","":"downstairs","session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3732026-05-19T05:11:48.287Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.616316757Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3742026-05-19T05:11:48.287Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.616377931Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"client":"1","":"downstairs","session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3752026-05-19T05:11:48.287Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.61639256Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3762026-05-19T05:11:48.287Z{"msg":"[0]R flush_numbers: [5322, 5322, 5322, 5322, 5322]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.616542905Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"downstairs","":"downstairs","session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3772026-05-19T05:11:48.287Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.616557484Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"downstairs","":"downstairs","session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3782026-05-19T05:11:48.287Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.616567993Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"downstairs","":"downstairs","session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3792026-05-19T05:11:48.287Z{"msg":"[1]R flush_numbers: [5322, 5322, 5322, 5322, 5322]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.616578562Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"downstairs","":"downstairs","session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3802026-05-19T05:11:48.287Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.6165927Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"downstairs","":"downstairs","session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3812026-05-19T05:11:48.287Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.616603379Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"downstairs","":"downstairs","session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3822026-05-19T05:11:48.287Z{"msg":"[2]R flush_numbers: [5322, 5322, 5322, 5322, 5322]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.616617588Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"downstairs","":"downstairs","session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3832026-05-19T05:11:48.287Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.616627777Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"downstairs","":"downstairs","session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3842026-05-19T05:11:48.287Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.616641886Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"downstairs","":"downstairs","session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3852026-05-19T05:11:48.287Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.616651915Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"downstairs","":"downstairs","session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3862026-05-19T05:11:48.287Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.616661504Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"downstairs","":"downstairs","session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3872026-05-19T05:11:48.287Z{"msg":"Next flush: 5323","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.616675153Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"downstairs","":"downstairs","session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3882026-05-19T05:11:48.287Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.616684872Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"":"downstairs","":"downstairs","session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3892026-05-19T05:11:48.287Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.61669856Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3902026-05-19T05:11:48.287Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.616707729Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"client":"0","":"downstairs","session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3912026-05-19T05:11:48.287Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.616721538Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"client":"1","":"downstairs","session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3922026-05-19T05:11:48.287Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.616731467Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"client":"2","":"downstairs","session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3932026-05-19T05:11:48.287Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.616745586Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3942026-05-19T05:11:48.287Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.616755155Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3952026-05-19T05:11:48.287Z{"msg":"6a0c33d0-dd78-4225-83d9-ff4b72a7718b is now active with session: bcd96321-2e2c-499e-98e3-2688765c8f5f","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.616769324Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3962026-05-19T05:11:48.287Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.616778453Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"session_id":"bcd96321-2e2c-499e-98e3-2688765c8f5f"}
3972026-05-19T05:11:48.287Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.616791871Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188}
3982026-05-19T05:11:48.287Z{"msg":"Activated sub_volume 6a0c33d0-dd78-4225-83d9-ff4b72a7718b","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.616799761Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188}
3992026-05-19T05:11:48.287Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.61681194Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"task":"crutest"}
4002026-05-19T05:11:48.287ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
4012026-05-19T05:11:48.287ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4022026-05-19T05:11:48.288Z{"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.617414072Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"task":"crutest"}
4032026-05-19T05:11:48.308ZRead and Verify all blocks (0..100 range:false)
4042026-05-19T05:11:48.355Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.68858364Z","----------------------------------------------------------------
4052026-05-19T05:11:48.355Zhostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"task Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4062026-05-19T05:11:48.355Z":"States:crutest" Active} Active
4072026-05-19T05:11:48.355Z Active
4082026-05-19T05:11:48.355Z{JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4092026-05-19T05:11:48.355Z"msg":"CLIENT: Tests done. All submitted work has been ACK'd" 1002 Acked Read 50,"v Done" Done: Done0 false,
4102026-05-19T05:11:48.356Z"name":"crucible","level":30 STATES DS:0 DS:1 DS:2 TOTAL
4112026-05-19T05:11:48.356Z Sent 0 0 0 0
4122026-05-19T05:11:48.356Z, Done " 1 time 1 ": 1 " 3
4132026-05-19T05:11:48.356Z2026-05-19T05:11:46.688621286Z Skipped " 0 0 , 0 " 0
4142026-05-19T05:11:48.356Zhostname":" Error 0 0 0 0
4152026-05-19T05:11:48.356Zw-01KRZA2C4BS76SB2900SK3KRE4","Last Flush: pid":1188Some(JobId(1001)) ,"Some(JobId(1001)) task"Some(JobId(1001)) :
4162026-05-19T05:11:48.356Z"Downstairs last five completed:crutest"}
4172026-05-19T05:11:48.356Z 1001 1000
4182026-05-19T05:11:48.356ZUpstairs last five completed: 1002 1001 1000
4192026-05-19T05:11:48.359Z----------------------------------------------------------------
4202026-05-19T05:11:48.359Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4212026-05-19T05:11:48.359ZStates: Active Active Active
4222026-05-19T05:11:48.359ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4232026-05-19T05:11:48.359Z 1002 Acked Read 50 Done Done Done false
4242026-05-19T05:11:48.359Z STATES DS:0 DS:1 DS:2 TOTAL
4252026-05-19T05:11:48.359Z Sent 0 0 0 0
4262026-05-19T05:11:48.359Z Done 1 1 1 3
4272026-05-19T05:11:48.359Z Skipped 0 0 0 0
4282026-05-19T05:11:48.359Z Error 0 0 0 0
4292026-05-19T05:11:48.359ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4302026-05-19T05:11:48.359ZDownstairs last five completed: 1001 1000
4312026-05-19T05:11:48.359ZUpstairs last five completed: 1002 1001 1000
4322026-05-19T05:11:48.359Z{"msg":"CLIENT: Up:0 ds:2 act:6","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:46.688744855Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"task":"crutest"}
4332026-05-19T05:11:52.369Z----------------------------------------------------------------
4342026-05-19T05:11:52.369Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4352026-05-19T05:11:52.369ZDownstairs last five completed: 1003 1002 1001 1000
4362026-05-19T05:11:52.369ZUpstairs last five completed: 1003 1002 1001 1000
4372026-05-19T05:11:52.373Z----------------------------------------------------------------
4382026-05-19T05:11:52.373Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4392026-05-19T05:11:52.373ZDownstairs last five completed: 1003 1002 1001 1000
4402026-05-19T05:11:52.373ZUpstairs last five completed: 1003 1002 1001 1000
4412026-05-19T05:11:52.373Z{"msg":"CLIENT: Up:0 ds:0 act:6","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.690413585Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"task":"crutest"}
4422026-05-19T05:11:52.373Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.690483379Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188,"task":"crutest"}
4432026-05-19T05:11:52.373Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-19T05:11:50.690502857Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1188}
4442026-05-19T05:11:52.373ZRepair part 1 passed
4452026-05-19T05:11:52.373Z
4462026-05-19T05:11:52.373ZKill the current downstairs
4472026-05-19T05:11:52.407ZDownstairs 2 stopped
4482026-05-19T05:11:52.407Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
4492026-05-19T05:11:52.424ZNow put back the original so we have a mismatch
4502026-05-19T05:11:52.424Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
4512026-05-19T05:11:52.424ZRestart downstairs with old directory
4522026-05-19T05:11:52.441Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4532026-05-19T05:11:52.445ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4542026-05-19T05:11:52.446Z 0 000-009 32 32 31 3999 3999 3998 F F F <---
4552026-05-19T05:11:52.446Z 1 010-019 32 32 31 3999 3999 3998 F F F <---
4562026-05-19T05:11:52.446Z 2 020-029 32 32 31 3999 3999 3998 F F F <---
4572026-05-19T05:11:52.446Z 3 030-039 32 32 31 3999 3999 3998 F F F <---
4582026-05-19T05:11:52.446Z 4 040-049 32 32 31 3999 3999 3998 F F F <---
4592026-05-19T05:11:52.446ZMax gen: 32, Max flush: 3999
4602026-05-19T05:11:52.446ZError: Difference in extent metadata found!
4612026-05-19T05:11:52.446Zdump test found error as expected
4622026-05-19T05:11:52.446Z
4632026-05-19T05:11:52.446Z
4642026-05-19T05:11:52.446Z/var/tmp/bins/crutest verify --range -g 33 -q --key ZWnKbMVewk/OQ6RT+cwMAypB2RTj2mH6fJ+xiu6b8Eg= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
4652026-05-19T05:11:52.460Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.787357904Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"task":"crutest"}
4662026-05-19T05:11:52.468Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.787741077Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"task":"crutest"}
4672026-05-19T05:11:52.468Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.788744691Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"task":"crutest"}
4682026-05-19T05:11:52.468Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.788996587Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196}
4692026-05-19T05:11:52.468Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c9d557ea5b0ac329d65f72d3a87737e706c99cd3\",\n git_commit_timestamp: \"2026-05-19T04:52:54.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.789058891Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196}
4702026-05-19T05:11:52.468Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.78907259Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196}
4712026-05-19T05:11:52.468Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.789085359Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196}
4722026-05-19T05:11:52.468Z{"msg":"Crucible cdcc62eb-5a46-4038-ae86-05b22541c77a has session id: e16c126a-f376-4bd7-a7db-476790fb4d14","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.789439915Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
4732026-05-19T05:11:52.468Z{"msg":"Upstairs opts: Upstairs UUID: cdcc62eb-5a46-4038-ae86-05b22541c77a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.789462363Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
4742026-05-19T05:11:52.468Z{"msg":"Crucible stats registered with UUID: cdcc62eb-5a46-4038-ae86-05b22541c77a","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.789476761Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
4752026-05-19T05:11:52.468Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-19T05:11:50.78948712Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
4762026-05-19T05:11:52.468Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.789632596Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
4772026-05-19T05:11:52.468Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.789653814Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
4782026-05-19T05:11:52.468Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.789668453Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
4792026-05-19T05:11:52.468Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.790599304Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"task":"crutest"}
4802026-05-19T05:11:52.468Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.790672117Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196}
4812026-05-19T05:11:52.468Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c9d557ea5b0ac329d65f72d3a87737e706c99cd3\",\n git_commit_timestamp: \"2026-05-19T04:52:54.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.790695555Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196}
4822026-05-19T05:11:52.468Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.790708094Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196}
4832026-05-19T05:11:52.468Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.790719523Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196}
4842026-05-19T05:11:52.469Z{"msg":"Crucible cdcc62eb-5a46-4038-ae86-05b22541c77a has session id: a8922839-af1a-474d-8ec4-730b6e1ac6f3","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.790901815Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
4852026-05-19T05:11:52.469Z{"msg":"Upstairs opts: Upstairs UUID: cdcc62eb-5a46-4038-ae86-05b22541c77a, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.790924153Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
4862026-05-19T05:11:52.469Z{"msg":"Crucible stats registered with UUID: cdcc62eb-5a46-4038-ae86-05b22541c77a","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.790938572Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
4872026-05-19T05:11:52.469Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-19T05:11:50.790947971Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
4882026-05-19T05:11:52.469Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.79096114Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
4892026-05-19T05:11:52.469Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.790971459Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
4902026-05-19T05:11:52.469Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.790982068Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
4912026-05-19T05:11:52.469Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.791097767Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"task":"crutest"}
4922026-05-19T05:11:52.469Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.791111735Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196}
4932026-05-19T05:11:52.469Z{"msg":"cdcc62eb-5a46-4038-ae86-05b22541c77a active request set","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.791120684Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
4942026-05-19T05:11:52.469Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.791142722Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"client":"0","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
4952026-05-19T05:11:52.469Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.791156881Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"client":"1","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
4962026-05-19T05:11:52.469Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.79116752Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"client":"2","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
4972026-05-19T05:11:52.469Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.791180979Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
4982026-05-19T05:11:52.469Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.791195177Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
4992026-05-19T05:11:52.469Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.791209576Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5002026-05-19T05:11:52.469Z{"msg":"ds_connection connected from Ok(127.0.0.1:46005)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.791235283Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5012026-05-19T05:11:52.469Z{"msg":"ds_connection connected from Ok(127.0.0.1:63893)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.791250342Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5022026-05-19T05:11:52.469Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-05-19T05:11:50.791324385Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5032026-05-19T05:11:52.469Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-05-19T05:11:50.791352792Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5042026-05-19T05:11:52.469Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-05-19T05:11:50.791366231Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5052026-05-19T05:11:52.469Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.79137523Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5062026-05-19T05:11:52.469Z{"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.791398708Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5072026-05-19T05:11:52.469Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.791869643Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"client":"0","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5082026-05-19T05:11:52.470Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.79190493Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"client":"1","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5092026-05-19T05:11:52.470Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.79200038Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5102026-05-19T05:11:52.470Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:50.792013869Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5112026-05-19T05:12:02.461Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.792200751Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5122026-05-19T05:12:02.461Z{"msg":"ds_connection connected from Ok(127.0.0.1:42612)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.792288863Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5132026-05-19T05:12:02.466Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.793506397Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"client":"2","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5142026-05-19T05:12:02.466Z{"msg":"[0]R flush_numbers: [3999, 3999, 3999, 3999, 3999]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.793775491Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"downstairs","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5152026-05-19T05:12:02.466Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.793801299Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"downstairs","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5162026-05-19T05:12:02.466Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.793818487Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"downstairs","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5172026-05-19T05:12:02.466Z{"msg":"[1]R flush_numbers: [3999, 3999, 3999, 3999, 3999]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.793842795Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"downstairs","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5182026-05-19T05:12:02.466Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.793858983Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"downstairs","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5192026-05-19T05:12:02.466Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.793882821Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"downstairs","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5202026-05-19T05:12:02.466Z{"msg":"[2]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.793899609Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"downstairs","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5212026-05-19T05:12:02.466Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.793922907Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"downstairs","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5222026-05-19T05:12:02.466Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.793942835Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"downstairs","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5232026-05-19T05:12:02.466Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.793966193Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"downstairs","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5242026-05-19T05:12:02.466Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.793982681Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"downstairs","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5252026-05-19T05:12:02.466Z{"msg":"Next flush: 4000","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.794005089Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"downstairs","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5262026-05-19T05:12:02.466Z{"msg":"extent 0 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.794028987Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5272026-05-19T05:12:02.466Z{"msg":"extent:0 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.794060304Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5282026-05-19T05:12:02.466Z{"msg":"found dest for source 0 for extent at index 0 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.794079482Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5292026-05-19T05:12:02.466Z{"msg":"extent 1 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.794109809Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5302026-05-19T05:12:02.466Z{"msg":"extent:1 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.794132917Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5312026-05-19T05:12:02.466Z{"msg":"found dest for source 0 for extent at index 1 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.794157495Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5322026-05-19T05:12:02.466Z{"msg":"extent 2 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.794181552Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5332026-05-19T05:12:02.467Z{"msg":"extent:2 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.79421141Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5342026-05-19T05:12:02.467Z{"msg":"found dest for source 0 for extent at index 2 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.794235997Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5352026-05-19T05:12:02.467Z{"msg":"extent 3 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.794259005Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5362026-05-19T05:12:02.467Z{"msg":"extent:3 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.794283433Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5372026-05-19T05:12:02.467Z{"msg":"found dest for source 0 for extent at index 3 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.79430863Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5382026-05-19T05:12:02.467Z{"msg":"extent 4 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.794331798Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5392026-05-19T05:12:02.467Z{"msg":"extent:4 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.794355346Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5402026-05-19T05:12:02.467Z{"msg":"found dest for source 0 for extent at index 4 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.794373484Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5412026-05-19T05:12:02.467Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.794398072Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"client":"0","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5422026-05-19T05:12:02.467Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.79441696Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"client":"1","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5432026-05-19T05:12:02.467Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.794435448Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"client":"2","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5442026-05-19T05:12:02.467Z{"msg":"Full repair list: {ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.794477214Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"downstairs","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5452026-05-19T05:12:02.467Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 4000, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 4000, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 4000, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 4000, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 4000, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.79462745Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"downstairs","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5462026-05-19T05:12:02.467Z{"msg":"starting reconciliation 40ccdc98-ba67-4ddc-b9fe-dabbea5eeed4: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.794646788Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"downstairs","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5472026-05-19T05:12:02.467Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.794663086Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"client":"2","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5482026-05-19T05:12:02.472Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.804315145Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"client":"2","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5492026-05-19T05:12:02.475Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.806555722Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"client":"2","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5502026-05-19T05:12:02.478Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.808516114Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"client":"2","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5512026-05-19T05:12:02.481Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.810486036Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"client":"2","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5522026-05-19T05:12:02.481Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.812207552Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"downstairs","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5532026-05-19T05:12:02.481Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.81223482Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"client":"0","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5542026-05-19T05:12:02.481Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.812249968Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"client":"1","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5552026-05-19T05:12:02.481Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.812264417Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"client":"2","":"downstairs","session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5562026-05-19T05:12:02.481Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.812278695Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5572026-05-19T05:12:02.481Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.812288344Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5582026-05-19T05:12:02.481Z{"msg":"cdcc62eb-5a46-4038-ae86-05b22541c77a is now active with session: e16c126a-f376-4bd7-a7db-476790fb4d14","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.812302833Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"session_id":"e16c126a-f376-4bd7-a7db-476790fb4d14"}
5592026-05-19T05:12:02.481Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.812316412Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196}
5602026-05-19T05:12:02.481Z{"msg":"Activated sub_volume cdcc62eb-5a46-4038-ae86-05b22541c77a","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.812324791Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196}
5612026-05-19T05:12:02.481Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.812375576Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196}
5622026-05-19T05:12:02.481Z{"msg":"cdcc62eb-5a46-4038-ae86-05b22541c77a active request set","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.812396984Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5632026-05-19T05:12:02.481Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.812410553Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"client":"0","":"downstairs","session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5642026-05-19T05:12:02.481Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.812420802Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"client":"1","":"downstairs","session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5652026-05-19T05:12:02.481Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.812430631Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"client":"2","":"downstairs","session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5662026-05-19T05:12:02.481Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.812444979Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5672026-05-19T05:12:02.481Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.812455289Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5682026-05-19T05:12:02.481Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.812469257Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5692026-05-19T05:12:02.481Z{"msg":"ds_connection connected from Ok(127.0.0.1:40914)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.812566078Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5702026-05-19T05:12:02.482Z{"msg":"ds_connection connected from Ok(127.0.0.1:63620)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.812642681Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5712026-05-19T05:12:02.482Z{"msg":"ds_connection connected from Ok(127.0.0.1:63487)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.812654659Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5722026-05-19T05:12:02.485Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.813136913Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"client":"0","":"downstairs","session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5732026-05-19T05:12:02.485Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.81316825Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"client":"1","":"downstairs","session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5742026-05-19T05:12:02.485Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.813185029Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"client":"2","":"downstairs","session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5752026-05-19T05:12:02.485Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.813343534Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5762026-05-19T05:12:02.485Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.81338704Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5772026-05-19T05:12:02.485Z{"msg":"[0]R flush_numbers: [5323, 5323, 5323, 5323, 5323]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.813459763Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"downstairs","":"downstairs","session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5782026-05-19T05:12:02.485Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.813476481Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"downstairs","":"downstairs","session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5792026-05-19T05:12:02.485Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.81348724Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"downstairs","":"downstairs","session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5802026-05-19T05:12:02.485Z{"msg":"[1]R flush_numbers: [5323, 5323, 5323, 5323, 5323]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.813501489Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"downstairs","":"downstairs","session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5812026-05-19T05:12:02.485Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.813511298Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"downstairs","":"downstairs","session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5822026-05-19T05:12:02.485Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.813526326Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"downstairs","":"downstairs","session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5832026-05-19T05:12:02.485Z{"msg":"[2]R flush_numbers: [5323, 5323, 5323, 5323, 5323]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.813536445Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"downstairs","":"downstairs","session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5842026-05-19T05:12:02.485Z{"msg":"[2]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.813550684Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"downstairs","":"downstairs","session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5852026-05-19T05:12:02.485Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.813560803Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"downstairs","":"downstairs","session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5862026-05-19T05:12:02.485Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.813570542Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"downstairs","":"downstairs","session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5872026-05-19T05:12:02.485Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.813584641Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"downstairs","":"downstairs","session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5882026-05-19T05:12:02.485Z{"msg":"Next flush: 5324","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.81359417Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"downstairs","":"downstairs","session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5892026-05-19T05:12:02.485Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.813607958Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"":"downstairs","":"downstairs","session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5902026-05-19T05:12:02.486Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.813617668Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5912026-05-19T05:12:02.486Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.813630656Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"client":"0","":"downstairs","session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5922026-05-19T05:12:02.486Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.813640595Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"client":"1","":"downstairs","session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5932026-05-19T05:12:02.486Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.813654594Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"client":"2","":"downstairs","session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5942026-05-19T05:12:02.486Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.813664343Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5952026-05-19T05:12:02.486Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.813677632Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5962026-05-19T05:12:02.486Z{"msg":"cdcc62eb-5a46-4038-ae86-05b22541c77a is now active with session: a8922839-af1a-474d-8ec4-730b6e1ac6f3","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.813687901Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5972026-05-19T05:12:02.486Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.81370101Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"session_id":"a8922839-af1a-474d-8ec4-730b6e1ac6f3"}
5982026-05-19T05:12:02.486Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.813710379Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196}
5992026-05-19T05:12:02.486Z{"msg":"Activated sub_volume cdcc62eb-5a46-4038-ae86-05b22541c77a","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.813722878Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196}
6002026-05-19T05:12:02.486Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.813731107Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"task":"crutest"}
6012026-05-19T05:12:02.486ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
6022026-05-19T05:12:02.486ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
6032026-05-19T05:12:02.486ZRead and Verify all blocks (0..100 range:true)
6042026-05-19T05:12:02.530Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.861506187Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"task":"crutest"}
6052026-05-19T05:12:02.530Z{"msg":"Verify test completed","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.861531944Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"task":"crutest"}
6062026-05-19T05:12:02.530Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.861546733Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"task":"crutest"}
6072026-05-19T05:12:02.530Z{"msg":"CLIENT: Tests done. All submitted work has been ACK'd","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.861555672Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"task":"crutest"}
6082026-05-19T05:12:02.530Z----------------------------------------------------------------
6092026-05-19T05:12:02.530Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6102026-05-19T05:12:02.530ZStates: Active Active Active
6112026-05-19T05:12:02.530ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6122026-05-19T05:12:02.533Z 1000 Acked Read 50 Done Done Done false
6132026-05-19T05:12:02.533Z STATES DS:0 DS:1 DS:2 TOTAL
6142026-05-19T05:12:02.533Z Sent 0 0 0 0
6152026-05-19T05:12:02.533Z Done 1 1 1 3
6162026-05-19T05:12:02.533Z Skipped 0 0 0 0
6172026-05-19T05:12:02.533Z Error 0 0 0 0
6182026-05-19T05:12:02.533ZLast Flush: None None None
6192026-05-19T05:12:02.533ZDownstairs last five completed:
6202026-05-19T05:12:02.533ZUpstairs last five completed: 1000
6212026-05-19T05:12:02.533Z----------------------------------------------------------------
6222026-05-19T05:12:02.533Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6232026-05-19T05:12:02.533ZStates: Active Active Active
6242026-05-19T05:12:02.534ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6252026-05-19T05:12:02.534Z 1000 Acked Read 50 Done Done Done false
6262026-05-19T05:12:02.534Z STATES DS:0 DS:1 DS:2 TOTAL
6272026-05-19T05:12:02.534Z Sent 0 0 0 0
6282026-05-19T05:12:02.534Z Done 1 1 1 3
6292026-05-19T05:12:02.534Z Skipped 0 0 0 0
6302026-05-19T05:12:02.534Z Error 0 0 0 0
6312026-05-19T05:12:02.534ZLast Flush: None None None
6322026-05-19T05:12:02.534ZDownstairs last five completed:
6332026-05-19T05:12:02.534ZUpstairs last five completed: 1000
6342026-05-19T05:12:02.534Z{"msg":"CLIENT: Up:0 ds:2 act:6","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:00.861699568Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"task":"crutest"}
6352026-05-19T05:12:06.532Z----------------------------------------------------------------
6362026-05-19T05:12:06.532Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6372026-05-19T05:12:06.532ZDownstairs last five completed: 1001 1000
6382026-05-19T05:12:06.532ZUpstairs last five completed: 1001 1000
6392026-05-19T05:12:06.532Z----------------------------------------------------------------
6402026-05-19T05:12:06.532Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6412026-05-19T05:12:06.532ZDownstairs last five completed: 1001 1000
6422026-05-19T05:12:06.532ZUpstairs last five completed: 1001 1000
6432026-05-19T05:12:06.536Z{"msg":"CLIENT: Up:0 ds:0 act:6","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:04.863305074Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"task":"crutest"}
6442026-05-19T05:12:06.536Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:12:04.86334782Z","hostname":"w-01KRZA2C4BS76SB2900SK3KRE4","pid":1196,"task":"crutest"}
6452026-05-19T05:12:06.539ZRepair part 2 passed
6462026-05-19T05:12:06.539Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
6472026-05-19T05:12:06.551ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6482026-05-19T05:12:06.551Z 0 000-009 32 32 32 3999 3999 3999 F F F
6492026-05-19T05:12:06.551Z 1 010-019 32 32 32 3999 3999 3999 F F F
6502026-05-19T05:12:06.551Z 2 020-029 32 32 32 3999 3999 3999 F F F
6512026-05-19T05:12:06.551Z 3 030-039 32 32 32 3999 3999 3999 F F F
6522026-05-19T05:12:06.551Z 4 040-049 32 32 32 3999 3999 3999 F F F
6532026-05-19T05:12:06.554ZMax gen: 32, Max flush: 3999
6542026-05-19T05:12:06.554Zdump test passed
6552026-05-19T05:12:06.554Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
6562026-05-19T05:12:06.564Z Extent 1
6572026-05-19T05:12:06.567ZGEN 32 32 32
6582026-05-19T05:12:06.567ZFLUSH_ID 3999 3999 3999
6592026-05-19T05:12:06.567ZDIRTY
6602026-05-19T05:12:06.567Z
6612026-05-19T05:12:06.567ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
6622026-05-19T05:12:06.567Z010 A A A A A A
6632026-05-19T05:12:06.567Z011 A A A A A A
6642026-05-19T05:12:06.567Z012 A A A A A A
6652026-05-19T05:12:06.567Z013 A A A A A A
6662026-05-19T05:12:06.571Z014 A A A A A A
6672026-05-19T05:12:06.571Z015 A A A A A A
6682026-05-19T05:12:06.571Z016 A A A A A A
6692026-05-19T05:12:06.571Z017 A A A A A A
6702026-05-19T05:12:06.571Z018 A A A A A A
6712026-05-19T05:12:06.571Z019 A A A A A A
6722026-05-19T05:12:06.575Zdump extent test passed
6732026-05-19T05:12:06.575Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
6742026-05-19T05:12:06.583ZExtent 2 Block in extent 0 Actual block 20
6752026-05-19T05:12:06.583Z
6762026-05-19T05:12:06.586Z DATA SHA256 VER
6772026-05-19T05:12:06.587Z------ ---------------------------------------------------------------- ---
6782026-05-19T05:12:06.587Z 0 59de8ded6fdb0875b0e7a313a1f0f224c2f756b6bf5e190ad4b3cd16f9592303 A
6792026-05-19T05:12:06.587Z 1 59de8ded6fdb0875b0e7a313a1f0f224c2f756b6bf5e190ad4b3cd16f9592303 A
6802026-05-19T05:12:06.587Z 2 59de8ded6fdb0875b0e7a313a1f0f224c2f756b6bf5e190ad4b3cd16f9592303 A
6812026-05-19T05:12:06.587Z
6822026-05-19T05:12:06.587ZNONCES 0 1 2 DIFF
6832026-05-19T05:12:06.587Z------ ------------------------ ------------------------ ------------------------ -----
6842026-05-19T05:12:06.587Z 0 d7ba1cfccce41bd1aa4fda49 d7ba1cfccce41bd1aa4fda49 d7ba1cfccce41bd1aa4fda49
6852026-05-19T05:12:06.587Z
6862026-05-19T05:12:06.587Z TAGS 0 1 2 DIFF
6872026-05-19T05:12:06.587Z------ -------------------------------- -------------------------------- -------------------------------- -----
6882026-05-19T05:12:06.587Z 0 385342ec426909f0c5aebf9e9b343211 385342ec426909f0c5aebf9e9b343211 385342ec426909f0c5aebf9e9b343211
6892026-05-19T05:12:06.587Z
6902026-05-19T05:12:06.587ZHASHES 0 1 2 DIFF
6912026-05-19T05:12:06.587Z------ ---------------- ---------------- ---------------- -----
6922026-05-19T05:12:06.587Z
6932026-05-19T05:12:06.590Zdump block test passed
6942026-05-19T05:12:06.590ZInitial upstairs tests have completed, stopping all downstairs
6952026-05-19T05:12:11.608ZCreating 7 larger downstairs regions
6962026-05-19T05:12:11.812ZStarting 7 downstairs
6972026-05-19T05:12:16.818Zdsc restarted at PID: 1212
6982026-05-19T05:12:16.915ZNow do the replace-reconcile test
6992026-05-19T05:12:16.928ZUsing 8870 for the replacement port
7002026-05-19T05:15:57.370ZCompleted test: replace-reconcile
7012026-05-19T05:15:57.370ZNow do the replace-before-active test
7022026-05-19T05:19:50.771ZCompleted test: replace-before-active
7032026-05-19T05:19:50.771ZAll tests have completed, stopping all downstairs
7042026-05-19T05:19:50.789Z
7052026-05-19T05:19:50.789ZAll Tests have passed
7062026-05-19T05:19:50.789Z11:41 Test duration
7072026-05-19T05:19:50.789Z
7082026-05-19T05:19:50.789Zreal 11:40.755967400
7092026-05-19T05:19:50.789Zuser 16:39.955800648
7102026-05-19T05:19:50.789Zsys 11.714000483
7112026-05-19T05:19:50.789Ztrap 0.425477979
7122026-05-19T05:19:50.789Ztflt 0.034825640
7132026-05-19T05:19:50.789Zdflt 0.026369426
7142026-05-19T05:19:50.789Zkflt 0.001155195
7152026-05-19T05:19:50.789Zlock 15:51:37.856869857
7162026-05-19T05:19:50.789Zslp 50:07.633598831
7172026-05-19T05:19:50.789Zlat 1:50.163774612
7182026-05-19T05:19:50.789Zstop 0.051785256
7192026-05-19T05:19:50.792Z+ echo 'test-up-2region-encrypted ends'
7202026-05-19T05:19:50.793Ztest-up-2region-encrypted ends
7212026-05-19T05:19:55.794Zprocess exited: duration 709824 ms, exit code 0
7222026-05-19T05:19:55.794Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
7232026-05-19T05:20:55.801Zexec warning: : stdout descriptor may be held open by a background process; giving up!
7242026-05-19T05:20:55.801Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
7252026-05-19T05:20:55.813Zfound 17 output files
7262026-05-19T05:20:55.813Zuploading: /tmp/test_up-build/dsc-out.txt (59015 bytes)
7272026-05-19T05:20:56.826Zuploaded: /tmp/test_up-build/dsc-out.txt
7282026-05-19T05:20:56.826Zuploading: /tmp/test_up-build/test_up_out.txt (23038048 bytes)
7292026-05-19T05:20:57.968Zuploaded: /tmp/test_up-build/test_up_out.txt
7302026-05-19T05:20:57.971Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (550964 bytes)
7312026-05-19T05:20:58.985Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
7322026-05-19T05:20:58.985Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (548322 bytes)
7332026-05-19T05:21:00.004Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
7342026-05-19T05:21:00.004Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (978689 bytes)
7352026-05-19T05:21:01.026Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
7362026-05-19T05:21:01.026Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (543057 bytes)
7372026-05-19T05:21:02.045Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
7382026-05-19T05:21:02.045Zuploading: /tmp/test_up-build/dsc/downstairs-8850.txt (2498 bytes)
7392026-05-19T05:21:03.055Zuploaded: /tmp/test_up-build/dsc/downstairs-8850.txt
7402026-05-19T05:21:03.055Zuploading: /tmp/test_up-build/dsc/downstairs-8860.txt (47329 bytes)
7412026-05-19T05:21:04.069Zuploaded: /tmp/test_up-build/dsc/downstairs-8860.txt
7422026-05-19T05:21:04.069Zuploading: /tmp/test_up-build/dsc/downstairs-8870.txt (3606819 bytes)
7432026-05-19T05:21:05.101Zuploaded: /tmp/test_up-build/dsc/downstairs-8870.txt
7442026-05-19T05:21:05.101Zuploading: /tmp/debug/df.txt (1270 bytes)
7452026-05-19T05:21:06.110Zuploaded: /tmp/debug/df.txt
7462026-05-19T05:21:06.110Zuploading: /tmp/debug/dtrace.txt (1035389 bytes)
7472026-05-19T05:21:07.133Zuploaded: /tmp/debug/dtrace.txt
7482026-05-19T05:21:07.133Zuploading: /tmp/debug/iostat.txt (227004 bytes)
7492026-05-19T05:21:07.143Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 227004 -> 230556
7502026-05-19T05:21:08.150Zuploaded: /tmp/debug/iostat.txt
7512026-05-19T05:21:08.150Zuploading: /tmp/debug/mpstat.txt (555528 bytes)
7522026-05-19T05:21:08.162Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 555528 -> 564953
7532026-05-19T05:21:09.170Zuploaded: /tmp/debug/mpstat.txt
7542026-05-19T05:21:09.170Zuploading: /tmp/debug/paging.txt (97192 bytes)
7552026-05-19T05:21:09.178Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 97192 -> 98904
7562026-05-19T05:21:10.186Zuploaded: /tmp/debug/paging.txt
7572026-05-19T05:21:10.186Zuploading: /tmp/debug/prstat.txt (1064090 bytes)
7582026-05-19T05:21:10.202Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 1064090 -> 1082188
7592026-05-19T05:21:11.209Zuploaded: /tmp/debug/prstat.txt
7602026-05-19T05:21:11.209Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
7612026-05-19T05:21:12.217Zuploaded: /tmp/debug/psrinfo.txt
7622026-05-19T05:21:12.217Zuploading: /tmp/debug/upstairs-info.txt (165648 bytes)
7632026-05-19T05:21:13.230Zuploaded: /tmp/debug/upstairs-info.txt