01KYJ9Q8H31747ZDJCVANY6FP7: test-up-2region-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01KYJ9QPTZSF5N92AD2NPTZ286

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12026-07-27T17:37:22.607Zjob dependencies complete; ready to run (waiting for 12 m 17 s)
22026-07-27T17:39:03.205Zjob assigned to worker 01KYJAECTPQ3WH533KDSPYCFMF [factory aws, i-05a6bdc87658e76fd] (queued for 1 m 40 s)
32026-07-27T17:39:09.729Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42026-07-27T17:39:11.397Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52026-07-27T17:39:11.400Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62026-07-27T17:39:13.140Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72026-07-27T17:39:13.143Zdownloading input: /input/build/work/bins/crutest.gz
82026-07-27T17:39:14.681Zdownloaded input: /input/build/work/bins/crutest.gz
92026-07-27T17:39:14.684Zdownloading input: /input/build/work/bins/dsc.gz
102026-07-27T17:39:15.471Zdownloaded input: /input/build/work/bins/dsc.gz
112026-07-27T17:39:15.475Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122026-07-27T17:39:15.502Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132026-07-27T17:39:15.505Zdownloading input: /input/build/work/scripts/test_ds.sh
142026-07-27T17:39:15.527Zdownloaded input: /input/build/work/scripts/test_ds.sh
152026-07-27T17:39:15.530Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162026-07-27T17:39:15.552Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172026-07-27T17:39:15.552Zdownloading input: /input/build/work/scripts/test_repair.sh
182026-07-27T17:39:15.582Zdownloaded input: /input/build/work/scripts/test_repair.sh
192026-07-27T17:39:15.582Zdownloading input: /input/build/work/scripts/test_replay.sh
202026-07-27T17:39:15.610Zdownloaded input: /input/build/work/scripts/test_replay.sh
212026-07-27T17:39:15.613Zdownloading input: /input/build/work/scripts/test_up.sh
222026-07-27T17:39:15.635Zdownloaded input: /input/build/work/scripts/test_up.sh
232026-07-27T17:39:15.635Zdownloading input: /input/build/work/scripts/upstairs_info.d
242026-07-27T17:39:15.661Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252026-07-27T17:39:15.664Zdownloading input: /input/build/tmp/cargo-test-out.log
262026-07-27T17:39:16.297Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272026-07-27T17:39:16.297Zstarting task 0: "setup"
282026-07-27T17:39:16.304Z++ uname -s
292026-07-27T17:39:16.307Z+ kern=SunOS
302026-07-27T17:39:16.308Z+ build_user=build
312026-07-27T17:39:16.308Z+ build_uid=12345
322026-07-27T17:39:16.308Z+ work_dir=/work
332026-07-27T17:39:16.308Z+ input_dir=/input
342026-07-27T17:39:16.308Z+ [[ 0 == 12345 ]]
352026-07-27T17:39:16.308Z+ case "$kern" in
362026-07-27T17:39:16.308Z+ groupadd -g 12345 build
372026-07-27T17:39:16.308Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382026-07-27T17:39:18.312Z+ zfs create -o mountpoint=/work rpool/work
392026-07-27T17:39:18.661Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402026-07-27T17:39:18.666Z+ home_fs=zfs
412026-07-27T17:39:18.667Z+ [[ zfs == autofs ]]
422026-07-27T17:39:18.667Z+ mkdir -p /home/build
432026-07-27T17:39:18.667Z+ chown build:build /home/build /work
442026-07-27T17:39:20.666Z+ chmod 0700 /home/build /work
452026-07-27T17:39:20.673Zprocess exited: duration 4369 ms, exit code 0
 
462026-07-27T17:39:20.683Zstarting task 1: "authentication"
472026-07-27T17:39:20.701Zprocess exited: duration 18 ms, exit code 0
 
482026-07-27T17:39:20.711Zstarting task 2: "build"
492026-07-27T17:39:20.718Z+ banner cores
502026-07-27T17:39:20.723Z
512026-07-27T17:39:20.723Z #### #### ##### ###### ####
522026-07-27T17:39:20.723Z # # # # # # # #
532026-07-27T17:39:20.723Z # # # # # ##### ####
542026-07-27T17:39:20.723Z # # # ##### # #
552026-07-27T17:39:20.723Z # # # # # # # # #
562026-07-27T17:39:20.723Z #### #### # # ###### ####
572026-07-27T17:39:20.723Z
582026-07-27T17:39:20.723Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
592026-07-27T17:39:20.731Z+ banner unpack
602026-07-27T17:39:20.731Z
612026-07-27T17:39:20.731Z # # # # ##### ## #### # #
622026-07-27T17:39:20.731Z # # ## # # # # # # # # #
632026-07-27T17:39:20.731Z # # # # # # # # # # ####
642026-07-27T17:39:20.731Z # # # # # ##### ###### # # #
652026-07-27T17:39:20.731Z # # # ## # # # # # # #
662026-07-27T17:39:20.731Z #### # # # # # #### # #
672026-07-27T17:39:20.731Z
682026-07-27T17:39:20.731Z+ mkdir -p /var/tmp/bins
692026-07-27T17:39:20.732Z+ for t in "$input/bins/"*.gz
702026-07-27T17:39:20.732Z++ basename /input/build/work/bins/crucible-downstairs.gz
712026-07-27T17:39:20.732Z+ b=crucible-downstairs.gz
722026-07-27T17:39:20.732Z+ b=crucible-downstairs
732026-07-27T17:39:20.732Z+ gunzip
742026-07-27T17:39:23.321Z+ chmod +x /var/tmp/bins/crucible-downstairs
752026-07-27T17:39:23.326Z+ for t in "$input/bins/"*.gz
762026-07-27T17:39:23.326Z++ basename /input/build/work/bins/crucible-hammer.gz
772026-07-27T17:39:23.326Z+ b=crucible-hammer.gz
782026-07-27T17:39:23.327Z+ b=crucible-hammer
792026-07-27T17:39:23.327Z+ gunzip
802026-07-27T17:39:25.733Z+ chmod +x /var/tmp/bins/crucible-hammer
812026-07-27T17:39:25.736Z+ for t in "$input/bins/"*.gz
822026-07-27T17:39:25.736Z++ basename /input/build/work/bins/crutest.gz
832026-07-27T17:39:25.736Z+ b=crutest.gz
842026-07-27T17:39:25.736Z+ b=crutest
852026-07-27T17:39:25.736Z+ gunzip
862026-07-27T17:39:28.268Z+ chmod +x /var/tmp/bins/crutest
872026-07-27T17:39:28.272Z+ for t in "$input/bins/"*.gz
882026-07-27T17:39:28.272Z++ basename /input/build/work/bins/dsc.gz
892026-07-27T17:39:28.272Z+ b=dsc.gz
902026-07-27T17:39:28.272Z+ b=dsc
912026-07-27T17:39:28.272Z+ gunzip
922026-07-27T17:39:29.401Z+ chmod +x /var/tmp/bins/dsc
932026-07-27T17:39:29.405Z+ export BINDIR=/var/tmp/bins
942026-07-27T17:39:29.405Z+ BINDIR=/var/tmp/bins
952026-07-27T17:39:29.405Z+ jobpid=1089
962026-07-27T17:39:29.405Z+ echo 'Setup debug logging'
972026-07-27T17:39:29.405Z+ mkdir /tmp/debug
982026-07-27T17:39:29.405ZSetup debug logging
992026-07-27T17:39:29.405Z+ sleep 3600
1002026-07-27T17:39:29.405Z+ psrinfo -v
1012026-07-27T17:39:29.409Z+ df -h
1022026-07-27T17:39:29.413Z+ prstat -d d -mLc 1
1032026-07-27T17:39:29.417Z+ iostat -T d -xn 1
1042026-07-27T17:39:29.417Z+ mpstat -T d 1
1052026-07-27T17:39:29.417Z+ vmstat -T d -p 1
1062026-07-27T17:39:29.417Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1072026-07-27T17:39:29.417Z+ banner test_up_2r_encrypted
1082026-07-27T17:39:29.417Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1092026-07-27T17:39:29.421Z #####
1102026-07-27T17:39:29.421Z ##### ###### #### ##### # # ##### # # #####
1112026-07-27T17:39:29.421Z # # # # # # # # # # #
1122026-07-27T17:39:29.421Z # ##### #### # # # # # ##### # #
1132026-07-27T17:39:29.421Z # # # # # # ##### # #####
1142026-07-27T17:39:29.421Z # # # # # # # # # # #
1152026-07-27T17:39:29.421Z # ###### #### # ####### #### # ####### ####### # #
1162026-07-27T17:39:29.421Z
1172026-07-27T17:39:29.421Z+ ptime -m bash /input/build/work/scripts/test_up.sh -r 2 -N encrypted
1182026-07-27T17:39:29.425Z/input/build/work
1192026-07-27T17:39:29.433ZUsing 2 region sets
1202026-07-27T17:39:29.433ZTurn off color for downstairs dump
1212026-07-27T17:39:29.445ZUpstairs using key: lBhHiO35CkoYqY7Na+Q+rFmmsWbohqMUT1I1j1E3vZ8=
1222026-07-27T17:39:29.448Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1232026-07-27T17:39:29.448ZCreating 6 downstairs regions
1242026-07-27T17:39:29.699ZStarting 6 downstairs
1252026-07-27T17:39:29.703Zdsc started at PID: 1137
1262026-07-27T17:39:34.789ZDisable automatic restart on all downstairs
1272026-07-27T17:39:34.803Z
1282026-07-27T17:39:34.803ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1292026-07-27T17:39:34.822ZRunning test: span
1302026-07-27T17:39:38.877ZCompleted test: span
1312026-07-27T17:39:43.894ZRunning test: big
1322026-07-27T17:39:48.191ZCompleted test: big
1332026-07-27T17:39:53.192ZRunning test: dep
1342026-07-27T17:40:12.936ZCompleted test: dep
1352026-07-27T17:40:17.969ZRunning test: balloon
1362026-07-27T17:42:45.211ZCompleted test: balloon
1372026-07-27T17:42:50.216ZRunning test: deactivate
1382026-07-27T17:42:54.579ZCompleted test: deactivate
1392026-07-27T17:42:59.585ZRunning hammer
1402026-07-27T17:43:11.754ZRun repair tests
1412026-07-27T17:43:11.757Z/var/tmp/bins/crutest fill -g 31 -q --key lBhHiO35CkoYqY7Na+Q+rFmmsWbohqMUT1I1j1E3vZ8= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1422026-07-27T17:43:11.775Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.914346108Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"task":"crutest"}
1432026-07-27T17:43:11.780Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.91473614Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"task":"crutest"}
1442026-07-27T17:43:11.780Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.915433313Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"task":"crutest"}
1452026-07-27T17:43:11.780Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.915679429Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180}
1462026-07-27T17:43:11.780Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"810bc819bfbfe43d9067f8b000b1765e3cf5b63d\",\n git_commit_timestamp: \"2026-07-27T17:23:45.000000000Z\",\n git_branch: \"alan/ro-read-just-one\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.915716036Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180}
1472026-07-27T17:43:11.780Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.915729815Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180}
1482026-07-27T17:43:11.780Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.915741863Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180}
1492026-07-27T17:43:11.780Z{"msg":"Crucible aa7df2e8-9632-481d-883c-7dd5f9656a77 has session id: 6645bd07-b976-4a62-a313-5f0938a67015","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.916120217Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1502026-07-27T17:43:11.780Z{"msg":"Upstairs opts: Upstairs UUID: aa7df2e8-9632-481d-883c-7dd5f9656a77, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.916148624Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1512026-07-27T17:43:11.780Z{"msg":"Crucible stats registered with UUID: aa7df2e8-9632-481d-883c-7dd5f9656a77","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.916159413Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1522026-07-27T17:43:11.780Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-27T17:43:09.916169002Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1532026-07-27T17:43:11.780Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.916306449Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"io task","client":"1","":"downstairs","session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1542026-07-27T17:43:11.780Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.916326187Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"io task","client":"0","":"downstairs","session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1552026-07-27T17:43:11.780Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.916336706Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"io task","client":"2","":"downstairs","session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1562026-07-27T17:43:11.781Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.917241049Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"task":"crutest"}
1572026-07-27T17:43:11.781Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.917302763Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180}
1582026-07-27T17:43:11.781Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"810bc819bfbfe43d9067f8b000b1765e3cf5b63d\",\n git_commit_timestamp: \"2026-07-27T17:23:45.000000000Z\",\n git_branch: \"alan/ro-read-just-one\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.917322461Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180}
1592026-07-27T17:43:11.781Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.91733143Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180}
1602026-07-27T17:43:11.781Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.917343289Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180}
1612026-07-27T17:43:11.781Z{"msg":"Crucible aa7df2e8-9632-481d-883c-7dd5f9656a77 has session id: cfd8a002-b479-468b-8c82-aabd51fc4f7f","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.917552898Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
1622026-07-27T17:43:11.781Z{"msg":"Upstairs opts: Upstairs UUID: aa7df2e8-9632-481d-883c-7dd5f9656a77, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.917588355Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
1632026-07-27T17:43:11.781Z{"msg":"Crucible stats registered with UUID: aa7df2e8-9632-481d-883c-7dd5f9656a77","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.917600164Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
1642026-07-27T17:43:11.781Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-27T17:43:09.917609343Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
1652026-07-27T17:43:11.781Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.917618842Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"io task","client":"0","":"downstairs","session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
1662026-07-27T17:43:11.781Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.917630211Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"io task","client":"1","":"downstairs","session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
1672026-07-27T17:43:11.781Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.91764118Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"io task","client":"2","":"downstairs","session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
1682026-07-27T17:43:11.781Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.917780526Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"task":"crutest"}
1692026-07-27T17:43:11.781Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.917792035Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180}
1702026-07-27T17:43:11.781Z{"msg":"aa7df2e8-9632-481d-883c-7dd5f9656a77 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.917831152Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1712026-07-27T17:43:11.781Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.917841311Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"client":"0","":"downstairs","session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1722026-07-27T17:43:11.781Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.91785178Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"client":"1","":"downstairs","session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1732026-07-27T17:43:11.781Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.917869968Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"client":"2","":"downstairs","session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1742026-07-27T17:43:11.781Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.917879727Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"io task","client":"2","":"downstairs","session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1752026-07-27T17:43:11.781Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.917893746Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"io task","client":"0","":"downstairs","session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1762026-07-27T17:43:11.781Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.917908354Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"io task","client":"1","":"downstairs","session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1772026-07-27T17:43:11.781Z{"msg":"ds_connection connected from Ok(127.0.0.1:61645)","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.917924823Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"io task","client":"2","":"downstairs","session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1782026-07-27T17:43:11.781Z{"msg":"ds_connection connected from Ok(127.0.0.1:34349)","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.917939701Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"io task","client":"0","":"downstairs","session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1792026-07-27T17:43:11.781Z{"msg":"ds_connection connected from Ok(127.0.0.1:54176)","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.917959959Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"io task","client":"1","":"downstairs","session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1802026-07-27T17:43:11.786Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.918696588Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"client":"1","":"downstairs","session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1812026-07-27T17:43:11.786Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.918727735Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"client":"0","":"downstairs","session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1822026-07-27T17:43:11.786Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.918740064Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"client":"2","":"downstairs","session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1832026-07-27T17:43:11.786Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.918907408Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1842026-07-27T17:43:11.786Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.918921476Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1852026-07-27T17:43:11.786Z{"msg":"[0]R flush_numbers: [3992, 3992, 3992, 3992, 3992]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.918956123Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"downstairs","":"downstairs","session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1862026-07-27T17:43:11.786Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.918970831Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"downstairs","":"downstairs","session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1872026-07-27T17:43:11.786Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.918981031Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"downstairs","":"downstairs","session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1882026-07-27T17:43:11.786Z{"msg":"[1]R flush_numbers: [3992, 3992, 3992, 3992, 3992]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.918991649Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"downstairs","":"downstairs","session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1892026-07-27T17:43:11.786Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.919005958Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"downstairs","":"downstairs","session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1902026-07-27T17:43:11.786Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.919016557Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"downstairs","":"downstairs","session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1912026-07-27T17:43:11.786Z{"msg":"[2]R flush_numbers: [3992, 3992, 3992, 3992, 3992]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.919031146Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"downstairs","":"downstairs","session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1922026-07-27T17:43:11.787Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.919041335Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"downstairs","":"downstairs","session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1932026-07-27T17:43:11.787Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.919055573Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"downstairs","":"downstairs","session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1942026-07-27T17:43:11.787Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.919065792Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"downstairs","":"downstairs","session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1952026-07-27T17:43:11.787Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.919075981Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"downstairs","":"downstairs","session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1962026-07-27T17:43:11.787Z{"msg":"Next flush: 3993","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.91908544Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"downstairs","":"downstairs","session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1972026-07-27T17:43:11.787Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.919095659Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"downstairs","":"downstairs","session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1982026-07-27T17:43:11.787Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.919110028Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
1992026-07-27T17:43:11.787Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.919119287Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"client":"0","":"downstairs","session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
2002026-07-27T17:43:11.787Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.919133446Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"client":"1","":"downstairs","session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
2012026-07-27T17:43:11.787Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.919143325Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"client":"2","":"downstairs","session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
2022026-07-27T17:43:11.787Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.919157533Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
2032026-07-27T17:43:11.787Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.919166773Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
2042026-07-27T17:43:11.787Z{"msg":"aa7df2e8-9632-481d-883c-7dd5f9656a77 is now active with session: 6645bd07-b976-4a62-a313-5f0938a67015","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.919181381Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
2052026-07-27T17:43:11.787Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.91918998Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"session_id":"6645bd07-b976-4a62-a313-5f0938a67015"}
2062026-07-27T17:43:11.788Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.919199609Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180}
2072026-07-27T17:43:11.788Z{"msg":"Activated sub_volume aa7df2e8-9632-481d-883c-7dd5f9656a77","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.919208029Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180}
2082026-07-27T17:43:11.788Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.919220667Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180}
2092026-07-27T17:43:11.788Z{"msg":"aa7df2e8-9632-481d-883c-7dd5f9656a77 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.919239396Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2102026-07-27T17:43:11.788Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.919252964Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"client":"0","":"downstairs","session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2112026-07-27T17:43:11.788Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.919262583Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"client":"1","":"downstairs","session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2122026-07-27T17:43:11.788Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.919280052Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"client":"2","":"downstairs","session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2132026-07-27T17:43:11.788Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.919289661Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"io task","client":"2","":"downstairs","session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2142026-07-27T17:43:11.788Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.919353855Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"io task","client":"0","":"downstairs","session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2152026-07-27T17:43:11.788Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.919423378Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"io task","client":"1","":"downstairs","session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2162026-07-27T17:43:11.788Z{"msg":"ds_connection connected from Ok(127.0.0.1:37951)","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.919442206Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"io task","client":"0","":"downstairs","session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2172026-07-27T17:43:11.788Z{"msg":"ds_connection connected from Ok(127.0.0.1:49929)","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.919453255Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"io task","client":"2","":"downstairs","session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2182026-07-27T17:43:11.788Z{"msg":"ds_connection connected from Ok(127.0.0.1:41729)","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.919463944Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"io task","client":"1","":"downstairs","session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2192026-07-27T17:43:11.788Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.919728018Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"client":"0","":"downstairs","session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2202026-07-27T17:43:11.788Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.91981182Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"client":"1","":"downstairs","session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2212026-07-27T17:43:11.788Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.919824669Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2222026-07-27T17:43:11.788Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.91991447Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"client":"2","":"downstairs","session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2232026-07-27T17:43:11.788Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.919998442Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2242026-07-27T17:43:11.788Z{"msg":"[0]R flush_numbers: [5313, 5312, 5311, 5312, 5307]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.920058336Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"downstairs","":"downstairs","session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2252026-07-27T17:43:11.788Z{"msg":"[0]R generation: [14, 14, 13, 14, 10]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.920087074Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"downstairs","":"downstairs","session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2262026-07-27T17:43:11.788Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.920098283Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"downstairs","":"downstairs","session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2272026-07-27T17:43:11.788Z{"msg":"[1]R flush_numbers: [5313, 5312, 5311, 5312, 5307]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.920113171Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"downstairs","":"downstairs","session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2282026-07-27T17:43:11.788Z{"msg":"[1]R generation: [14, 14, 13, 14, 10]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.92012369Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"downstairs","":"downstairs","session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2292026-07-27T17:43:11.788Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.920138449Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"downstairs","":"downstairs","session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2302026-07-27T17:43:11.788Z{"msg":"[2]R flush_numbers: [5313, 5312, 5311, 5312, 5307]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.920148498Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"downstairs","":"downstairs","session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2312026-07-27T17:43:11.788Z{"msg":"[2]R generation: [14, 14, 13, 14, 10]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.920158757Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"downstairs","":"downstairs","session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2322026-07-27T17:43:11.788Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.920169206Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"downstairs","":"downstairs","session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2332026-07-27T17:43:11.788Z{"msg":"Max found gen is 15","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.920183334Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"downstairs","":"downstairs","session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2342026-07-27T17:43:11.788Z{"msg":"Generation requested: 31 >= found:15","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.920193563Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"downstairs","":"downstairs","session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2352026-07-27T17:43:11.788Z{"msg":"Next flush: 5314","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.920207302Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"downstairs","":"downstairs","session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2362026-07-27T17:43:11.788Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.920217261Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"":"downstairs","":"downstairs","session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2372026-07-27T17:43:11.788Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.92022729Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2382026-07-27T17:43:11.788Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.920240479Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"client":"0","":"downstairs","session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2392026-07-27T17:43:11.788Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.920250208Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"client":"1","":"downstairs","session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2402026-07-27T17:43:11.788Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.920259767Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"client":"2","":"downstairs","session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2412026-07-27T17:43:11.788Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.920269736Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2422026-07-27T17:43:11.788Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.920287244Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2432026-07-27T17:43:11.789Z{"msg":"aa7df2e8-9632-481d-883c-7dd5f9656a77 is now active with session: cfd8a002-b479-468b-8c82-aabd51fc4f7f","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.920297273Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2442026-07-27T17:43:11.789Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.920310902Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"session_id":"cfd8a002-b479-468b-8c82-aabd51fc4f7f"}
2452026-07-27T17:43:11.789Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.920320651Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180}
2462026-07-27T17:43:11.789Z{"msg":"Activated sub_volume aa7df2e8-9632-481d-883c-7dd5f9656a77","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.92033287Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180}
2472026-07-27T17:43:11.789Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.920340879Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"task":"crutest"}
2482026-07-27T17:43:11.789ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
2492026-07-27T17:43:11.789Z{"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.9205432Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"task":"crutest"}
2502026-07-27T17:43:11.806ZRead and Verify all blocks (0..100 range:false)
2512026-07-27T17:43:11.854Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30----------------------------------------------------------------
2522026-07-27T17:43:11.854Z,"time":"2026-07-27T17:43:09.993461247Z","hostname":" Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2532026-07-27T17:43:11.854Zw-01KYJAECTPQ3WH533KDSPYCFMF"States:,"pid": Active1180 Active Active
2542026-07-27T17:43:11.854Z,"JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2552026-07-27T17:43:11.854Ztask":"crutest"}
2562026-07-27T17:43:11.858Z 1002 Acked Read 50{ Done" Donemsg Done" false
2572026-07-27T17:43:11.858Z:" STATES DS:0 DS:1 DS:2 TOTAL
2582026-07-27T17:43:11.858ZCLIENT: Tests done. All submitted work has been ACK'd Sent "," 0 v": 0 0 0 0
2592026-07-27T17:43:11.858Z," Done name" 1 :" 1 1 crucible 3
2602026-07-27T17:43:11.858Z" Skipped ," 0 level 0 " 0 : 0
2612026-07-27T17:43:11.858Z30 Error 0 0 0 0
2622026-07-27T17:43:11.859ZLast Flush: Some(JobId(1001)) ,"Some(JobId(1001)) timeSome(JobId(1001)) ":"
2632026-07-27T17:43:11.859Z2026-07-27T17:43:09.993502213ZDownstairs last five completed:","hostname":" 1001w-01KYJAECTPQ3WH533KDSPYCFMF 1000"
2642026-07-27T17:43:11.859Z,"pid":1180,"taskUpstairs last five completed: ":" 1002crutest" 1001 1000}
2652026-07-27T17:43:11.859Z
2662026-07-27T17:43:11.859Z----------------------------------------------------------------
2672026-07-27T17:43:11.859Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2682026-07-27T17:43:11.859ZStates: Active Active Active
2692026-07-27T17:43:11.859ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2702026-07-27T17:43:11.859Z 1002 Acked Read 50 Done Done Done false
2712026-07-27T17:43:11.859Z STATES DS:0 DS:1 DS:2 TOTAL
2722026-07-27T17:43:11.859Z Sent 0 0 0 0
2732026-07-27T17:43:11.859Z Done 1 1 1 3
2742026-07-27T17:43:11.859Z Skipped 0 0 0 0
2752026-07-27T17:43:11.859Z Error 0 0 0 0
2762026-07-27T17:43:11.859ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2772026-07-27T17:43:11.859ZDownstairs last five completed: 1001 1000
2782026-07-27T17:43:11.859ZUpstairs last five completed: 1002 1001 1000
2792026-07-27T17:43:11.859Z{"msg":"CLIENT: Up:0 ds:2 act:6","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:09.993582516Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"task":"crutest"}
2802026-07-27T17:43:15.856Z----------------------------------------------------------------
2812026-07-27T17:43:15.856Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2822026-07-27T17:43:15.856ZDownstairs last five completed: 1003 1002 1001 1000
2832026-07-27T17:43:15.856ZUpstairs last five completed: 1003 1002 1001 1000
2842026-07-27T17:43:15.856Z----------------------------------------------------------------
2852026-07-27T17:43:15.856Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2862026-07-27T17:43:15.856ZDownstairs last five completed: 1003 1002 1001 1000
2872026-07-27T17:43:15.856ZUpstairs last five completed: 1003 1002 1001 1000
2882026-07-27T17:43:15.859Z{"msg":"CLIENT: Up:0 ds:0 act:6","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:13.995135004Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"task":"crutest"}
2892026-07-27T17:43:15.859Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:13.995167241Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180,"task":"crutest"}
2902026-07-27T17:43:15.860Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-27T17:43:13.995212476Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1180}
2912026-07-27T17:43:15.864ZRepair setup passed
2922026-07-27T17:43:15.867ZCopy the region for /var/tmp/test_up-build/8830
2932026-07-27T17:43:15.867Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2942026-07-27T17:43:15.870Z/var/tmp/bins/crutest fill -g 32 -q --key lBhHiO35CkoYqY7Na+Q+rFmmsWbohqMUT1I1j1E3vZ8= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2952026-07-27T17:43:15.887Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.026183885Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"task":"crutest"}
2962026-07-27T17:43:15.891Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.026463088Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"task":"crutest"}
2972026-07-27T17:43:15.891Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.027142772Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"task":"crutest"}
2982026-07-27T17:43:15.891Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.027389789Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186}
2992026-07-27T17:43:15.891Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"810bc819bfbfe43d9067f8b000b1765e3cf5b63d\",\n git_commit_timestamp: \"2026-07-27T17:23:45.000000000Z\",\n git_branch: \"alan/ro-read-just-one\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.027418316Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186}
3002026-07-27T17:43:15.891Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.027431415Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186}
3012026-07-27T17:43:15.891Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.027443673Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186}
3022026-07-27T17:43:15.891Z{"msg":"Crucible 86f8b7bd-6ada-4c76-ab48-e4444cf1f3e9 has session id: f256d593-38f8-4855-9674-fc5e3af27a36","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.027886941Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3032026-07-27T17:43:15.891Z{"msg":"Upstairs opts: Upstairs UUID: 86f8b7bd-6ada-4c76-ab48-e4444cf1f3e9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.027924237Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3042026-07-27T17:43:15.891Z{"msg":"Crucible stats registered with UUID: 86f8b7bd-6ada-4c76-ab48-e4444cf1f3e9","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.027940105Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3052026-07-27T17:43:15.891Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-27T17:43:14.027949805Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3062026-07-27T17:43:15.891Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.028040886Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"io task","client":"1","":"downstairs","session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3072026-07-27T17:43:15.891Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.028062384Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"io task","client":"0","":"downstairs","session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3082026-07-27T17:43:15.891Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.028072783Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"io task","client":"2","":"downstairs","session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3092026-07-27T17:43:15.891Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.029164277Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"task":"crutest"}
3102026-07-27T17:43:15.891Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.029216442Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186}
3112026-07-27T17:43:15.891Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"810bc819bfbfe43d9067f8b000b1765e3cf5b63d\",\n git_commit_timestamp: \"2026-07-27T17:23:45.000000000Z\",\n git_branch: \"alan/ro-read-just-one\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.02924179Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186}
3122026-07-27T17:43:15.892Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.029255248Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186}
3132026-07-27T17:43:15.892Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.029267307Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186}
3142026-07-27T17:43:15.892Z{"msg":"Crucible 86f8b7bd-6ada-4c76-ab48-e4444cf1f3e9 has session id: 1e1ceb8d-f1bf-454b-9d73-405b25df9e81","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.029473567Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3152026-07-27T17:43:15.892Z{"msg":"Upstairs opts: Upstairs UUID: 86f8b7bd-6ada-4c76-ab48-e4444cf1f3e9, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.029534331Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3162026-07-27T17:43:15.892Z{"msg":"Crucible stats registered with UUID: 86f8b7bd-6ada-4c76-ab48-e4444cf1f3e9","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.029566118Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3172026-07-27T17:43:15.898Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-27T17:43:14.029576507Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3182026-07-27T17:43:15.898Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.029590376Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"io task","client":"0","":"downstairs","session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3192026-07-27T17:43:15.898Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.029601295Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"io task","client":"2","":"downstairs","session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3202026-07-27T17:43:15.898Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.029612124Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"io task","client":"1","":"downstairs","session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3212026-07-27T17:43:15.898Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.029785417Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"task":"crutest"}
3222026-07-27T17:43:15.898Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.029811785Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186}
3232026-07-27T17:43:15.898Z{"msg":"86f8b7bd-6ada-4c76-ab48-e4444cf1f3e9 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.02985614Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3242026-07-27T17:43:15.898Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.029883698Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"client":"0","":"downstairs","session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3252026-07-27T17:43:15.898Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.029902016Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"client":"1","":"downstairs","session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3262026-07-27T17:43:15.898Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.029917215Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"client":"2","":"downstairs","session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3272026-07-27T17:43:15.898Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.029926994Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"io task","client":"0","":"downstairs","session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3282026-07-27T17:43:15.898Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.029937683Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"io task","client":"2","":"downstairs","session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3292026-07-27T17:43:15.898Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.029947822Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"io task","client":"1","":"downstairs","session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3302026-07-27T17:43:15.898Z{"msg":"ds_connection connected from Ok(127.0.0.1:48730)","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.029980108Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"io task","client":"0","":"downstairs","session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3312026-07-27T17:43:15.898Z{"msg":"ds_connection connected from Ok(127.0.0.1:58016)","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.029995017Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"io task","client":"2","":"downstairs","session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3322026-07-27T17:43:15.898Z{"msg":"ds_connection connected from Ok(127.0.0.1:55786)","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.030030584Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"io task","client":"1","":"downstairs","session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3332026-07-27T17:43:15.898Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.030707208Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"client":"1","":"downstairs","session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3342026-07-27T17:43:15.898Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.030751874Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"client":"0","":"downstairs","session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3352026-07-27T17:43:15.898Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.030771732Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"client":"2","":"downstairs","session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3362026-07-27T17:43:15.898Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.030916488Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3372026-07-27T17:43:15.898Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.030931147Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3382026-07-27T17:43:15.898Z{"msg":"[0]R flush_numbers: [3993, 3993, 3993, 3993, 3993]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.030968553Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"downstairs","":"downstairs","session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3392026-07-27T17:43:15.898Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.030983462Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"downstairs","":"downstairs","session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3402026-07-27T17:43:15.898Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.030993911Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"downstairs","":"downstairs","session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3412026-07-27T17:43:15.898Z{"msg":"[1]R flush_numbers: [3993, 3993, 3993, 3993, 3993]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031008319Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"downstairs","":"downstairs","session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3422026-07-27T17:43:15.899Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031018308Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"downstairs","":"downstairs","session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3432026-07-27T17:43:15.899Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031032477Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"downstairs","":"downstairs","session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3442026-07-27T17:43:15.899Z{"msg":"[2]R flush_numbers: [3993, 3993, 3993, 3993, 3993]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031042906Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"downstairs","":"downstairs","session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3452026-07-27T17:43:15.899Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031057194Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"downstairs","":"downstairs","session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3462026-07-27T17:43:15.899Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031067703Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"downstairs","":"downstairs","session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3472026-07-27T17:43:15.899Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031081512Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"downstairs","":"downstairs","session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3482026-07-27T17:43:15.899Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031091281Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"downstairs","":"downstairs","session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3492026-07-27T17:43:15.899Z{"msg":"Next flush: 3994","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.03110485Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"downstairs","":"downstairs","session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3502026-07-27T17:43:15.899Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031114809Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"downstairs","":"downstairs","session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3512026-07-27T17:43:15.899Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031128797Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3522026-07-27T17:43:15.899Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031137417Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"client":"0","":"downstairs","session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3532026-07-27T17:43:15.899Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031147916Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"client":"1","":"downstairs","session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3542026-07-27T17:43:15.899Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031161874Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"client":"2","":"downstairs","session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3552026-07-27T17:43:15.899Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031179533Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3562026-07-27T17:43:15.899Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031192921Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3572026-07-27T17:43:15.899Z{"msg":"86f8b7bd-6ada-4c76-ab48-e4444cf1f3e9 is now active with session: f256d593-38f8-4855-9674-fc5e3af27a36","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.03120321Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3582026-07-27T17:43:15.900Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031219539Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"session_id":"f256d593-38f8-4855-9674-fc5e3af27a36"}
3592026-07-27T17:43:15.900Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.03130956Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186}
3602026-07-27T17:43:15.900Z{"msg":"Activated sub_volume 86f8b7bd-6ada-4c76-ab48-e4444cf1f3e9","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031328838Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186}
3612026-07-27T17:43:15.900Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031341667Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186}
3622026-07-27T17:43:15.900Z{"msg":"86f8b7bd-6ada-4c76-ab48-e4444cf1f3e9 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031350256Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3632026-07-27T17:43:15.900Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031359675Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"client":"0","":"downstairs","session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3642026-07-27T17:43:15.900Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031369894Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"client":"1","":"downstairs","session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3652026-07-27T17:43:15.900Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031384033Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"client":"2","":"downstairs","session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3662026-07-27T17:43:15.900Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031393802Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"io task","client":"2","":"downstairs","session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3672026-07-27T17:43:15.900Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031408261Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"io task","client":"0","":"downstairs","session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3682026-07-27T17:43:15.900Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.03141821Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"io task","client":"1","":"downstairs","session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3692026-07-27T17:43:15.900Z{"msg":"ds_connection connected from Ok(127.0.0.1:40425)","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031428389Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"io task","client":"0","":"downstairs","session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3702026-07-27T17:43:15.900Z{"msg":"ds_connection connected from Ok(127.0.0.1:39328)","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031442917Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"io task","client":"2","":"downstairs","session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3712026-07-27T17:43:15.900Z{"msg":"ds_connection connected from Ok(127.0.0.1:53429)","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031460945Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"io task","client":"1","":"downstairs","session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3722026-07-27T17:43:15.900Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031561226Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"client":"0","":"downstairs","session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3732026-07-27T17:43:15.900Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031601382Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"client":"1","":"downstairs","session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3742026-07-27T17:43:15.900Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031691403Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3752026-07-27T17:43:15.900Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031746828Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"client":"2","":"downstairs","session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3762026-07-27T17:43:15.900Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031803122Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3772026-07-27T17:43:15.900Z{"msg":"[0]R flush_numbers: [5314, 5314, 5314, 5314, 5314]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031886824Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"downstairs","":"downstairs","session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3782026-07-27T17:43:15.900Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031901833Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"downstairs","":"downstairs","session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3792026-07-27T17:43:15.900Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031911682Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"downstairs","":"downstairs","session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3802026-07-27T17:43:15.900Z{"msg":"[1]R flush_numbers: [5314, 5314, 5314, 5314, 5314]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.03192615Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"downstairs","":"downstairs","session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3812026-07-27T17:43:15.900Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.0319364Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"downstairs","":"downstairs","session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3822026-07-27T17:43:15.900Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031951218Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"downstairs","":"downstairs","session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3832026-07-27T17:43:15.900Z{"msg":"[2]R flush_numbers: [5314, 5314, 5314, 5314, 5314]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031961377Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"downstairs","":"downstairs","session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3842026-07-27T17:43:15.901Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031976016Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"downstairs","":"downstairs","session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3852026-07-27T17:43:15.901Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031986085Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"downstairs","":"downstairs","session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3862026-07-27T17:43:15.901Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.031999983Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"downstairs","":"downstairs","session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3872026-07-27T17:43:15.901Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.032010042Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"downstairs","":"downstairs","session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3882026-07-27T17:43:15.901Z{"msg":"Next flush: 5315","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.032023771Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"downstairs","":"downstairs","session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3892026-07-27T17:43:15.901Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.03203341Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"":"downstairs","":"downstairs","session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3902026-07-27T17:43:15.901Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.032047639Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3912026-07-27T17:43:15.901Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.032056758Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"client":"0","":"downstairs","session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3922026-07-27T17:43:15.901Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.032070687Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"client":"1","":"downstairs","session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3932026-07-27T17:43:15.901Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.032080236Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"client":"2","":"downstairs","session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3942026-07-27T17:43:15.901Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.032094334Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3952026-07-27T17:43:15.902Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.032103593Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3962026-07-27T17:43:15.902Z{"msg":"86f8b7bd-6ada-4c76-ab48-e4444cf1f3e9 is now active with session: 1e1ceb8d-f1bf-454b-9d73-405b25df9e81","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.032118292Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3972026-07-27T17:43:15.902Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.032126891Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"session_id":"1e1ceb8d-f1bf-454b-9d73-405b25df9e81"}
3982026-07-27T17:43:15.902Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.03214043Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186}
3992026-07-27T17:43:15.902Z{"msg":"Activated sub_volume 86f8b7bd-6ada-4c76-ab48-e4444cf1f3e9","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.032148719Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186}
4002026-07-27T17:43:15.902Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.032157218Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"task":"crutest"}
4012026-07-27T17:43:15.902ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
4022026-07-27T17:43:15.902ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4032026-07-27T17:43:15.902Z{"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.032598606Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"task":"crutest"}
4042026-07-27T17:43:15.918ZRead and Verify all blocks (0..100 range:false)
4052026-07-27T17:43:15.966Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.105123331Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"task":"crutest"}
4062026-07-27T17:43:15.966Z{----------------------------------------------------------------
4072026-07-27T17:43:15.966Z"msg":"CLIENT: Tests done. All submitted work has been ACK'd","v":0,"name":"crucible"," Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4082026-07-27T17:43:15.966ZlevelStates:":30 Active Active Active
4092026-07-27T17:43:15.967ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4102026-07-27T17:43:15.967Z,"time":"2026-07-27T17:43:14.105151409Z","hostname":" 1002 Acked Read 50w-01KYJAECTPQ3WH533KDSPYCFMF"," Donepid" Done: Done false1186
4112026-07-27T17:43:15.967Z,"task":"crutest"}
4122026-07-27T17:43:15.967Z STATES DS:0 DS:1 DS:2 TOTAL
4132026-07-27T17:43:15.967Z Sent 0 0 0 0
4142026-07-27T17:43:15.967Z Done 1 1 1 3
4152026-07-27T17:43:15.967Z Skipped 0 0 0 0
4162026-07-27T17:43:15.967Z Error 0 0 0 0
4172026-07-27T17:43:15.967ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4182026-07-27T17:43:15.967ZDownstairs last five completed: 1001 1000
4192026-07-27T17:43:15.967ZUpstairs last five completed: 1002 1001 1000
4202026-07-27T17:43:15.971Z----------------------------------------------------------------
4212026-07-27T17:43:15.971Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4222026-07-27T17:43:15.971ZStates: Active Active Active
4232026-07-27T17:43:15.971ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4242026-07-27T17:43:15.971Z 1002 Acked Read 50 Done Done Done false
4252026-07-27T17:43:15.971Z STATES DS:0 DS:1 DS:2 TOTAL
4262026-07-27T17:43:15.971Z Sent 0 0 0 0
4272026-07-27T17:43:15.971Z Done 1 1 1 3
4282026-07-27T17:43:15.971Z Skipped 0 0 0 0
4292026-07-27T17:43:15.971Z Error 0 0 0 0
4302026-07-27T17:43:15.971ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4312026-07-27T17:43:15.971ZDownstairs last five completed: 1001 1000
4322026-07-27T17:43:15.971ZUpstairs last five completed: 1002 1001 1000
4332026-07-27T17:43:15.971Z{"msg":"CLIENT: Up:0 ds:2 act:6","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:14.105263258Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"task":"crutest"}
4342026-07-27T17:43:19.967Z----------------------------------------------------------------
4352026-07-27T17:43:19.967Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4362026-07-27T17:43:19.967ZDownstairs last five completed: 1003 1002 1001 1000
4372026-07-27T17:43:19.967ZUpstairs last five completed: 1003 1002 1001 1000
4382026-07-27T17:43:19.967Z----------------------------------------------------------------
4392026-07-27T17:43:19.967Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4402026-07-27T17:43:19.967ZDownstairs last five completed: 1003 1002 1001 1000
4412026-07-27T17:43:19.968ZUpstairs last five completed: 1003 1002 1001 1000
4422026-07-27T17:43:19.973Z{"msg":"CLIENT: Up:0 ds:0 act:6","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.105518641Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"task":"crutest"}
4432026-07-27T17:43:19.973Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.105560927Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1186,"task":"crutest"}
4442026-07-27T17:43:19.976ZRepair part 1 passed
4452026-07-27T17:43:19.976Z
4462026-07-27T17:43:19.976ZKill the current downstairs
4472026-07-27T17:43:20.015ZDownstairs 2 stopped
4482026-07-27T17:43:20.015Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
4492026-07-27T17:43:20.018ZNow put back the original so we have a mismatch
4502026-07-27T17:43:20.018Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
4512026-07-27T17:43:20.018ZRestart downstairs with old directory
4522026-07-27T17:43:20.031Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4532026-07-27T17:43:20.042ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4542026-07-27T17:43:20.046Z 0 000-009 32 32 31 3994 3994 3993 F F F <---
4552026-07-27T17:43:20.046Z 1 010-019 32 32 31 3994 3994 3993 F F F <---
4562026-07-27T17:43:20.046Z 2 020-029 32 32 31 3994 3994 3993 F F F <---
4572026-07-27T17:43:20.046Z 3 030-039 32 32 31 3994 3994 3993 F F F <---
4582026-07-27T17:43:20.046Z 4 040-049 32 32 31 3994 3994 3993 F F F <---
4592026-07-27T17:43:20.046ZMax gen: 32, Max flush: 3994
4602026-07-27T17:43:20.046ZError: Difference in extent metadata found!
4612026-07-27T17:43:20.046Zdump test found error as expected
4622026-07-27T17:43:20.046Z
4632026-07-27T17:43:20.046Z
4642026-07-27T17:43:20.046Z/var/tmp/bins/crutest verify --range -g 33 -q --key lBhHiO35CkoYqY7Na+Q+rFmmsWbohqMUT1I1j1E3vZ8= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
4652026-07-27T17:43:20.063Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.201188392Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"task":"crutest"}
4662026-07-27T17:43:20.067Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.201527179Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"task":"crutest"}
4672026-07-27T17:43:20.067Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.202143639Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"task":"crutest"}
4682026-07-27T17:43:20.067Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.202393655Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194}
4692026-07-27T17:43:20.067Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"810bc819bfbfe43d9067f8b000b1765e3cf5b63d\",\n git_commit_timestamp: \"2026-07-27T17:23:45.000000000Z\",\n git_branch: \"alan/ro-read-just-one\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.202421952Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194}
4702026-07-27T17:43:20.067Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.202434761Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194}
4712026-07-27T17:43:20.067Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.20244691Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194}
4722026-07-27T17:43:20.067Z{"msg":"Crucible ff85cd18-d378-4842-b025-69c626f39a33 has session id: e4ca47ab-1e7f-4060-bcfb-fb4dea335f11","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.20286347Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
4732026-07-27T17:43:20.067Z{"msg":"Upstairs opts: Upstairs UUID: ff85cd18-d378-4842-b025-69c626f39a33, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.202886188Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
4742026-07-27T17:43:20.067Z{"msg":"Crucible stats registered with UUID: ff85cd18-d378-4842-b025-69c626f39a33","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.202900556Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
4752026-07-27T17:43:20.067Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-27T17:43:18.202909895Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
4762026-07-27T17:43:20.067Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.203037423Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"io task","client":"1","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
4772026-07-27T17:43:20.067Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.203059351Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"io task","client":"0","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
4782026-07-27T17:43:20.067Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.20306989Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"io task","client":"2","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
4792026-07-27T17:43:20.067Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.204270034Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"task":"crutest"}
4802026-07-27T17:43:20.067Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.204331628Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194}
4812026-07-27T17:43:20.067Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"810bc819bfbfe43d9067f8b000b1765e3cf5b63d\",\n git_commit_timestamp: \"2026-07-27T17:23:45.000000000Z\",\n git_branch: \"alan/ro-read-just-one\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.204351526Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194}
4822026-07-27T17:43:20.067Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.204360055Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194}
4832026-07-27T17:43:20.067Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.204371694Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194}
4842026-07-27T17:43:20.068Z{"msg":"Crucible ff85cd18-d378-4842-b025-69c626f39a33 has session id: cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.204598212Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
4852026-07-27T17:43:20.068Z{"msg":"Upstairs opts: Upstairs UUID: ff85cd18-d378-4842-b025-69c626f39a33, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.204613091Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
4862026-07-27T17:43:20.068Z{"msg":"Crucible stats registered with UUID: ff85cd18-d378-4842-b025-69c626f39a33","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.20462363Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
4872026-07-27T17:43:20.068Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-27T17:43:18.204633179Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
4882026-07-27T17:43:20.068Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.204643428Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"io task","client":"0","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
4892026-07-27T17:43:20.068Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.204654217Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"io task","client":"1","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
4902026-07-27T17:43:20.068Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.204690223Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"io task","client":"2","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
4912026-07-27T17:43:20.068Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.204795003Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"task":"crutest"}
4922026-07-27T17:43:20.068Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.204837209Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194}
4932026-07-27T17:43:20.068Z{"msg":"ff85cd18-d378-4842-b025-69c626f39a33 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.204870266Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
4942026-07-27T17:43:20.068Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.204880395Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"client":"0","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
4952026-07-27T17:43:20.068Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.204891844Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"client":"1","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
4962026-07-27T17:43:20.068Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.204902513Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"client":"2","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
4972026-07-27T17:43:20.068Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.204912172Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"io task","client":"2","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
4982026-07-27T17:43:20.068Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.204922381Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"io task","client":"0","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
4992026-07-27T17:43:20.068Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.20493283Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"io task","client":"1","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5002026-07-27T17:43:20.068Z{"msg":"ds_connection connected from Ok(127.0.0.1:32841)","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.204943969Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"io task","client":"1","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5012026-07-27T17:43:20.068Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-07-27T17:43:18.204997654Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"io task","client":"2","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5022026-07-27T17:43:20.068Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-07-27T17:43:18.205010782Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"io task","client":"2","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5032026-07-27T17:43:20.068Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-07-27T17:43:18.205023311Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5042026-07-27T17:43:20.068Z{"msg":"ds_connection connected from Ok(127.0.0.1:54928)","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.20503262Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"io task","client":"0","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5052026-07-27T17:43:20.068Z{"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.205145119Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"io task","client":"2","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5062026-07-27T17:43:20.068Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.205172687Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"io task","client":"2","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5072026-07-27T17:43:20.072Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.205594296Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"client":"1","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5082026-07-27T17:43:20.072Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.205614014Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"client":"0","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5092026-07-27T17:43:20.072Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.205747531Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5102026-07-27T17:43:20.072Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:18.20576157Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5112026-07-27T17:43:30.068Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.205788637Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"io task","client":"2","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5122026-07-27T17:43:30.069Z{"msg":"ds_connection connected from Ok(127.0.0.1:49449)","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.205856061Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"io task","client":"2","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5132026-07-27T17:43:30.074Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.206955265Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"client":"2","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5142026-07-27T17:43:30.074Z{"msg":"[0]R flush_numbers: [3994, 3994, 3994, 3994, 3994]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.207178373Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"downstairs","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5152026-07-27T17:43:30.074Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.207194211Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"downstairs","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5162026-07-27T17:43:30.074Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.207204531Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"downstairs","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5172026-07-27T17:43:30.074Z{"msg":"[1]R flush_numbers: [3994, 3994, 3994, 3994, 3994]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.207219359Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"downstairs","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5182026-07-27T17:43:30.074Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.207229328Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"downstairs","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5192026-07-27T17:43:30.074Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.207239577Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"downstairs","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5202026-07-27T17:43:30.074Z{"msg":"[2]R flush_numbers: [3993, 3993, 3993, 3993, 3993]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.207253896Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"downstairs","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5212026-07-27T17:43:30.074Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.207263625Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"downstairs","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5222026-07-27T17:43:30.074Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.207277833Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"downstairs","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5232026-07-27T17:43:30.074Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.207287093Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"downstairs","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5242026-07-27T17:43:30.074Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.207301531Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"downstairs","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5252026-07-27T17:43:30.075Z{"msg":"Next flush: 3995","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.20731071Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"downstairs","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5262026-07-27T17:43:30.075Z{"msg":"extent 0 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3994, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3994, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3993, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.207329248Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"mend","":"downstairs","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5272026-07-27T17:43:30.075Z{"msg":"extent:0 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3994, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3994, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3993, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.207348537Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"mend","":"downstairs","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5282026-07-27T17:43:30.075Z{"msg":"found dest for source 0 for extent at index 0 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.207359726Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"mend","":"downstairs","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5292026-07-27T17:43:30.075Z{"msg":"extent 1 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3994, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3994, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3993, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.207378154Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"mend","":"downstairs","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5302026-07-27T17:43:30.075Z{"msg":"extent:1 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3994, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3994, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3993, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.207396352Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"mend","":"downstairs","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5312026-07-27T17:43:30.075Z{"msg":"found dest for source 0 for extent at index 1 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.207411221Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"mend","":"downstairs","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5322026-07-27T17:43:30.075Z{"msg":"extent 2 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3994, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3994, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3993, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.207425009Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"mend","":"downstairs","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5332026-07-27T17:43:30.075Z{"msg":"extent:2 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3994, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3994, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3993, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.207443067Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"mend","":"downstairs","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5342026-07-27T17:43:30.075Z{"msg":"found dest for source 0 for extent at index 2 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.207453996Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"mend","":"downstairs","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5352026-07-27T17:43:30.075Z{"msg":"extent 3 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3994, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3994, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3993, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.207472155Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"mend","":"downstairs","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5362026-07-27T17:43:30.076Z{"msg":"extent:3 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3994, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3994, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3993, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.207486243Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"mend","":"downstairs","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5372026-07-27T17:43:30.076Z{"msg":"found dest for source 0 for extent at index 3 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.207497182Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"mend","":"downstairs","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5382026-07-27T17:43:30.076Z{"msg":"extent 4 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3994, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3994, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3993, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.20751537Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"mend","":"downstairs","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5392026-07-27T17:43:30.076Z{"msg":"extent:4 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3994, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3994, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3993, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.207528879Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"mend","":"downstairs","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5402026-07-27T17:43:30.076Z{"msg":"found dest for source 0 for extent at index 4 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.207543758Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"mend","":"downstairs","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5412026-07-27T17:43:30.076Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.207561576Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"client":"0","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5422026-07-27T17:43:30.076Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.207576405Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"client":"1","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5432026-07-27T17:43:30.076Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.207586484Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"client":"2","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5442026-07-27T17:43:30.076Z{"msg":"Full repair list: {ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.207616501Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"downstairs","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5452026-07-27T17:43:30.076Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 3995, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 3995, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 3995, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 3995, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 3995, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.207709892Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"downstairs","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5462026-07-27T17:43:30.076Z{"msg":"starting reconciliation 8ebe7e46-506f-4947-900f-42f432d8f195: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.207721611Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"downstairs","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5472026-07-27T17:43:30.076Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.207756917Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"client":"2","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5482026-07-27T17:43:30.080Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.217506626Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"client":"2","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5492026-07-27T17:43:30.083Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.219541379Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"client":"2","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5502026-07-27T17:43:30.086Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.221448025Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"client":"2","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5512026-07-27T17:43:30.086Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.223321794Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"client":"2","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5522026-07-27T17:43:30.091Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.22512665Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"downstairs","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5532026-07-27T17:43:30.091Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.225154047Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"client":"0","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5542026-07-27T17:43:30.091Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.225164506Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"client":"1","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5552026-07-27T17:43:30.091Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.225179105Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"client":"2","":"downstairs","session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5562026-07-27T17:43:30.091Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.225188854Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5572026-07-27T17:43:30.091Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.225198263Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5582026-07-27T17:43:30.091Z{"msg":"ff85cd18-d378-4842-b025-69c626f39a33 is now active with session: e4ca47ab-1e7f-4060-bcfb-fb4dea335f11","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.225212851Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"session_id":"e4ca47ab-1e7f-4060-bcfb-fb4dea335f11"}
5592026-07-27T17:43:30.091Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.22522208Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194}
5602026-07-27T17:43:30.091Z{"msg":"Activated sub_volume ff85cd18-d378-4842-b025-69c626f39a33","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.225234719Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194}
5612026-07-27T17:43:30.091Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.225286634Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194}
5622026-07-27T17:43:30.091Z{"msg":"ff85cd18-d378-4842-b025-69c626f39a33 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.225299783Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5632026-07-27T17:43:30.091Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.225309052Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"client":"0","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5642026-07-27T17:43:30.091Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.225323371Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"client":"1","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5652026-07-27T17:43:30.091Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.22533285Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"client":"2","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5662026-07-27T17:43:30.091Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.225346668Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"io task","client":"2","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5672026-07-27T17:43:30.091Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.225357287Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"io task","client":"0","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5682026-07-27T17:43:30.091Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.225371236Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"io task","client":"1","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5692026-07-27T17:43:30.091Z{"msg":"ds_connection connected from Ok(127.0.0.1:54502)","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.225381665Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"io task","client":"2","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5702026-07-27T17:43:30.092Z{"msg":"ds_connection connected from Ok(127.0.0.1:42310)","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.225396154Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"io task","client":"0","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5712026-07-27T17:43:30.092Z{"msg":"ds_connection connected from Ok(127.0.0.1:54492)","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.225406773Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"io task","client":"1","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5722026-07-27T17:43:30.092Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.2257475Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"client":"1","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5732026-07-27T17:43:30.092Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.225781486Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"client":"0","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5742026-07-27T17:43:30.092Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.225831152Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"client":"2","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5752026-07-27T17:43:30.092Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.225877737Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5762026-07-27T17:43:30.092Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.225926002Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5772026-07-27T17:43:30.092Z{"msg":"[0]R flush_numbers: [5315, 5315, 5315, 5315, 5315]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.22594675Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"downstairs","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5782026-07-27T17:43:30.092Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.225961269Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"downstairs","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5792026-07-27T17:43:30.092Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.225971498Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"downstairs","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5802026-07-27T17:43:30.092Z{"msg":"[1]R flush_numbers: [5315, 5315, 5315, 5315, 5315]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.225986017Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"downstairs","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5812026-07-27T17:43:30.092Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.226012844Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"downstairs","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5822026-07-27T17:43:30.092Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.226027263Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"downstairs","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5832026-07-27T17:43:30.093Z{"msg":"[2]R flush_numbers: [5315, 5315, 5315, 5315, 5315]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.226037172Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"downstairs","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5842026-07-27T17:43:30.093Z{"msg":"[2]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.22605114Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"downstairs","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5852026-07-27T17:43:30.093Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.226065569Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"downstairs","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5862026-07-27T17:43:30.093Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.226075348Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"downstairs","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5872026-07-27T17:43:30.093Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.226089527Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"downstairs","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5882026-07-27T17:43:30.093Z{"msg":"Next flush: 5316","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.226103195Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"downstairs","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5892026-07-27T17:43:30.093Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.226112714Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"downstairs","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5902026-07-27T17:43:30.093Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.226122813Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5912026-07-27T17:43:30.093Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.226131683Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"client":"0","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5922026-07-27T17:43:30.093Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.226141672Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"client":"1","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5932026-07-27T17:43:30.093Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.22615578Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"client":"2","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5942026-07-27T17:43:30.093Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.226165689Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5952026-07-27T17:43:30.093Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.226179278Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5962026-07-27T17:43:30.093Z{"msg":"ff85cd18-d378-4842-b025-69c626f39a33 is now active with session: cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.226194007Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5972026-07-27T17:43:30.093Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.226207195Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
5982026-07-27T17:43:30.093Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.226216374Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194}
5992026-07-27T17:43:30.093Z{"msg":"Activated sub_volume ff85cd18-d378-4842-b025-69c626f39a33","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.226228873Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194}
6002026-07-27T17:43:30.093Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.226237032Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"task":"crutest"}
6012026-07-27T17:43:30.093ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
6022026-07-27T17:43:30.093ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
6032026-07-27T17:43:30.093ZRead and Verify all blocks (0..100 range:true)
6042026-07-27T17:43:30.137Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.274734429Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"task":"crutest"}
6052026-07-27T17:43:30.137Z{"msg":"Verify test completed","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.274758756Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"task":"crutest"}
6062026-07-27T17:43:30.140Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.274790093Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"task":"crutest"}
6072026-07-27T17:43:30.141Z{"msg":"CLIENT: Tests done. All submitted work has been ACK'd","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.274803682Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"task":"crutest"}
6082026-07-27T17:43:30.141Z----------------------------------------------------------------
6092026-07-27T17:43:30.141Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6102026-07-27T17:43:30.141ZStates: Active Active Active
6112026-07-27T17:43:30.141ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6122026-07-27T17:43:30.141Z 1000 Acked Read 50 Done Done Done false
6132026-07-27T17:43:30.141Z STATES DS:0 DS:1 DS:2 TOTAL
6142026-07-27T17:43:30.141Z Sent 0 0 0 0
6152026-07-27T17:43:30.141Z Done 1 1 1 3
6162026-07-27T17:43:30.141Z Skipped 0 0 0 0
6172026-07-27T17:43:30.141Z Error 0 0 0 0
6182026-07-27T17:43:30.141ZLast Flush: None None None
6192026-07-27T17:43:30.141ZDownstairs last five completed:
6202026-07-27T17:43:30.141ZUpstairs last five completed: 1000
6212026-07-27T17:43:30.142Z----------------------------------------------------------------
6222026-07-27T17:43:30.142Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6232026-07-27T17:43:30.142ZStates: Active Active Active
6242026-07-27T17:43:30.142ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6252026-07-27T17:43:30.142Z 1000 Acked Read 50 Done Done Done false
6262026-07-27T17:43:30.142Z STATES DS:0 DS:1 DS:2 TOTAL
6272026-07-27T17:43:30.142Z Sent 0 0 0 0
6282026-07-27T17:43:30.142Z Done 1 1 1 3
6292026-07-27T17:43:30.142Z Skipped 0 0 0 0
6302026-07-27T17:43:30.142Z Error 0 0 0 0
6312026-07-27T17:43:30.142ZLast Flush: None None None
6322026-07-27T17:43:30.142ZDownstairs last five completed:
6332026-07-27T17:43:30.142ZUpstairs last five completed: 1000
6342026-07-27T17:43:30.142Z{"msg":"CLIENT: Up:0 ds:2 act:6","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:28.274962717Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"task":"crutest"}
6352026-07-27T17:43:34.139Z----------------------------------------------------------------
6362026-07-27T17:43:34.139Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6372026-07-27T17:43:34.139ZDownstairs last five completed: 1001 1000
6382026-07-27T17:43:34.139ZUpstairs last five completed: 1001 1000
6392026-07-27T17:43:34.142Z----------------------------------------------------------------
6402026-07-27T17:43:34.142Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6412026-07-27T17:43:34.142ZDownstairs last five completed: 1001 1000
6422026-07-27T17:43:34.142ZUpstairs last five completed: 1001 1000
6432026-07-27T17:43:34.142Z{"msg":"CLIENT: Up:0 ds:0 act:6","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:32.276193916Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"task":"crutest"}
6442026-07-27T17:43:34.142Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:32.276244771Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"task":"crutest"}
6452026-07-27T17:43:34.143Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-27T17:43:32.276264689Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194}
6462026-07-27T17:43:34.143Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-27T17:43:32.276272798Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194}
6472026-07-27T17:43:34.143Z{"msg":"client request queue closed unexpectedly; is the program exiting?","v":0,"name":"crucible","level":40,"time":"2026-07-27T17:43:32.276337772Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"io task","client":"1","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
6482026-07-27T17:43:34.143Z{"msg":"client task was cancelled without us; hopefully the program is exiting","v":0,"name":"crucible","level":40,"time":"2026-07-27T17:43:32.27635185Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"io task","client":"2","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
6492026-07-27T17:43:34.143Z{"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2026-07-27T17:43:32.276363079Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"io task","client":"1","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
6502026-07-27T17:43:34.143Z{"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2026-07-27T17:43:32.276374318Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"io task","client":"1","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
6512026-07-27T17:43:34.143Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:32.276384947Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"io task","client":"1","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
6522026-07-27T17:43:34.143Z{"msg":"client task is sending Done(ReceiveTaskCancelled)","v":0,"name":"crucible","level":40,"time":"2026-07-27T17:43:32.276396116Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"io task","client":"2","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
6532026-07-27T17:43:34.143Z{"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2026-07-27T17:43:32.276406905Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"io task","client":"2","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
6542026-07-27T17:43:34.143Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-07-27T17:43:32.276417524Z","hostname":"w-01KYJAECTPQ3WH533KDSPYCFMF","pid":1194,"":"io task","client":"2","":"downstairs","session_id":"cc7877e3-8eee-4a3a-89e2-b4ba59fcc84c"}
6552026-07-27T17:43:34.146ZRepair part 2 passed
6562026-07-27T17:43:34.146Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
6572026-07-27T17:43:34.157ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6582026-07-27T17:43:34.157Z 0 000-009 32 32 32 3994 3994 3994 F F F
6592026-07-27T17:43:34.158Z 1 010-019 32 32 32 3994 3994 3994 F F F
6602026-07-27T17:43:34.162Z 2 020-029 32 32 32 3994 3994 3994 F F F
6612026-07-27T17:43:34.162Z 3 030-039 32 32 32 3994 3994 3994 F F F
6622026-07-27T17:43:34.162Z 4 040-049 32 32 32 3994 3994 3994 F F F
6632026-07-27T17:43:34.162ZMax gen: 32, Max flush: 3994
6642026-07-27T17:43:34.162Zdump test passed
6652026-07-27T17:43:34.162Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
6662026-07-27T17:43:34.170Z Extent 1
6672026-07-27T17:43:34.174ZGEN 32 32 32
6682026-07-27T17:43:34.174ZFLUSH_ID 3994 3994 3994
6692026-07-27T17:43:34.174ZDIRTY
6702026-07-27T17:43:34.174Z
6712026-07-27T17:43:34.174ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
6722026-07-27T17:43:34.174Z010 A A A A A A
6732026-07-27T17:43:34.174Z011 A A A A A A
6742026-07-27T17:43:34.175Z012 A A A A A A
6752026-07-27T17:43:34.175Z013 A A A A A A
6762026-07-27T17:43:34.178Z014 A A A A A A
6772026-07-27T17:43:34.178Z015 A A A A A A
6782026-07-27T17:43:34.178Z016 A A A A A A
6792026-07-27T17:43:34.178Z017 A A A A A A
6802026-07-27T17:43:34.178Z018 A A A A A A
6812026-07-27T17:43:34.178Z019 A A A A A A
6822026-07-27T17:43:34.181Zdump extent test passed
6832026-07-27T17:43:34.181Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
6842026-07-27T17:43:34.189ZExtent 2 Block in extent 0 Actual block 20
6852026-07-27T17:43:34.189Z
6862026-07-27T17:43:34.192Z DATA SHA256 VER
6872026-07-27T17:43:34.192Z------ ---------------------------------------------------------------- ---
6882026-07-27T17:43:34.192Z 0 1cb860b71a1f521340fb52d5080123070d7cd24a8e39d92b91d124b0683fbea9 A
6892026-07-27T17:43:34.192Z 1 1cb860b71a1f521340fb52d5080123070d7cd24a8e39d92b91d124b0683fbea9 A
6902026-07-27T17:43:34.192Z 2 1cb860b71a1f521340fb52d5080123070d7cd24a8e39d92b91d124b0683fbea9 A
6912026-07-27T17:43:34.192Z
6922026-07-27T17:43:34.192ZNONCES 0 1 2 DIFF
6932026-07-27T17:43:34.192Z------ ------------------------ ------------------------ ------------------------ -----
6942026-07-27T17:43:34.192Z 0 6d419c609c1abe579c706ed1 6d419c609c1abe579c706ed1 6d419c609c1abe579c706ed1
6952026-07-27T17:43:34.192Z
6962026-07-27T17:43:34.192Z TAGS 0 1 2 DIFF
6972026-07-27T17:43:34.192Z------ -------------------------------- -------------------------------- -------------------------------- -----
6982026-07-27T17:43:34.192Z 0 2c9302c1009cb3c983424b9ecf2709a6 2c9302c1009cb3c983424b9ecf2709a6 2c9302c1009cb3c983424b9ecf2709a6
6992026-07-27T17:43:34.192Z
7002026-07-27T17:43:34.193ZHASHES 0 1 2 DIFF
7012026-07-27T17:43:34.193Z------ ---------------- ---------------- ---------------- -----
7022026-07-27T17:43:34.193Z
7032026-07-27T17:43:34.196Zdump block test passed
7042026-07-27T17:43:34.196ZInitial upstairs tests have completed, stopping all downstairs
7052026-07-27T17:43:39.213ZCreating 7 larger downstairs regions
7062026-07-27T17:43:39.421ZStarting 7 downstairs
7072026-07-27T17:43:44.427Zdsc restarted at PID: 1210
7082026-07-27T17:43:44.525ZNow do the replace-reconcile test
7092026-07-27T17:43:44.538ZUsing 8870 for the replacement port
7102026-07-27T17:47:33.184ZCompleted test: replace-reconcile
7112026-07-27T17:47:33.188ZNow do the replace-before-active test
7122026-07-27T17:51:27.862ZCompleted test: replace-before-active
7132026-07-27T17:51:27.862ZAll tests have completed, stopping all downstairs
7142026-07-27T17:51:27.879Z
7152026-07-27T17:51:27.880ZAll Tests have passed
7162026-07-27T17:51:27.880Z11:58 Test duration
7172026-07-27T17:51:27.880Z
7182026-07-27T17:51:27.880Zreal 11:58.382506757
7192026-07-27T17:51:27.880Zuser 16:58.583242928
7202026-07-27T17:51:27.880Zsys 11.670473830
7212026-07-27T17:51:27.880Ztrap 0.434483137
7222026-07-27T17:51:27.880Ztflt 0.039589729
7232026-07-27T17:51:27.880Zdflt 0.014631041
7242026-07-27T17:51:27.880Zkflt 0.000981392
7252026-07-27T17:51:27.880Zlock 16:26:30.512930575
7262026-07-27T17:51:27.880Zslp 51:44.822477573
7272026-07-27T17:51:27.880Zlat 1:44.221582974
7282026-07-27T17:51:27.880Zstop 0.020944419
7292026-07-27T17:51:27.880Z+ echo 'test-up-2region-encrypted ends'
7302026-07-27T17:51:27.880Ztest-up-2region-encrypted ends
7312026-07-27T17:51:32.882Zprocess exited: duration 727090 ms, exit code 0
7322026-07-27T17:51:32.882Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
7332026-07-27T17:52:32.916Zexec warning: : stdout descriptor may be held open by a background process; giving up!
7342026-07-27T17:52:32.917Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
7352026-07-27T17:52:32.930Zfound 17 output files
7362026-07-27T17:52:32.930Zuploading: /tmp/test_up-build/dsc-out.txt (59451 bytes)
7372026-07-27T17:52:33.947Zuploaded: /tmp/test_up-build/dsc-out.txt
7382026-07-27T17:52:33.947Zuploading: /tmp/test_up-build/test_up_out.txt (23113998 bytes)
7392026-07-27T17:52:35.257Zuploaded: /tmp/test_up-build/test_up_out.txt
7402026-07-27T17:52:35.257Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (550951 bytes)
7412026-07-27T17:52:36.291Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
7422026-07-27T17:52:36.291Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (548302 bytes)
7432026-07-27T17:52:37.315Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
7442026-07-27T17:52:37.315Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (978683 bytes)
7452026-07-27T17:52:41.993Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
7462026-07-27T17:52:41.993Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (543057 bytes)
7472026-07-27T17:52:43.021Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
7482026-07-27T17:52:43.021Zuploading: /tmp/test_up-build/dsc/downstairs-8850.txt (2498 bytes)
7492026-07-27T17:52:44.030Zuploaded: /tmp/test_up-build/dsc/downstairs-8850.txt
7502026-07-27T17:52:44.030Zuploading: /tmp/test_up-build/dsc/downstairs-8860.txt (47334 bytes)
7512026-07-27T17:52:45.042Zuploaded: /tmp/test_up-build/dsc/downstairs-8860.txt
7522026-07-27T17:52:45.042Zuploading: /tmp/test_up-build/dsc/downstairs-8870.txt (3586259 bytes)
7532026-07-27T17:52:46.080Zuploaded: /tmp/test_up-build/dsc/downstairs-8870.txt
7542026-07-27T17:52:46.080Zuploading: /tmp/debug/df.txt (1270 bytes)
7552026-07-27T17:52:47.092Zuploaded: /tmp/debug/df.txt
7562026-07-27T17:52:47.092Zuploading: /tmp/debug/dtrace.txt (1134656 bytes)
7572026-07-27T17:52:48.113Zuploaded: /tmp/debug/dtrace.txt
7582026-07-27T17:52:48.113Zuploading: /tmp/debug/iostat.txt (233110 bytes)
7592026-07-27T17:52:48.123Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 233110 -> 237565
7602026-07-27T17:52:49.130Zuploaded: /tmp/debug/iostat.txt
7612026-07-27T17:52:49.130Zuploading: /tmp/debug/mpstat.txt (569354 bytes)
7622026-07-27T17:52:49.146Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 569354 -> 580970
7632026-07-27T17:52:50.153Zuploaded: /tmp/debug/mpstat.txt
7642026-07-27T17:52:50.153Zuploading: /tmp/debug/paging.txt (100290 bytes)
7652026-07-27T17:52:50.160Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 100290 -> 102510
7662026-07-27T17:52:51.167Zuploaded: /tmp/debug/paging.txt
7672026-07-27T17:52:51.167Zuploading: /tmp/debug/prstat.txt (1092270 bytes)
7682026-07-27T17:52:51.184Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 1092270 -> 1115511
7692026-07-27T17:52:52.191Zuploaded: /tmp/debug/prstat.txt
7702026-07-27T17:52:52.191Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
7712026-07-27T17:52:53.200Zuploaded: /tmp/debug/psrinfo.txt
7722026-07-27T17:52:53.200Zuploading: /tmp/debug/upstairs-info.txt (171360 bytes)
7732026-07-27T17:52:54.217Zuploaded: /tmp/debug/upstairs-info.txt