01KY0EQKZ3EJYPWAGM3W0P8APK: test-up-2region-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01KY0ER3WFX6G02BTG61NRREAB

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12026-07-20T19:18:20.252Zjob dependencies complete; ready to run (waiting for 11 m 58 s)
22026-07-20T19:19:17.717Zjob assigned to worker 01KY0FE5VTQK6S1ZS8T3K93BSW [factory aws, i-058e7fe40745d2d68] (queued for 57 s)
32026-07-20T19:19:24.108Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42026-07-20T19:19:25.722Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52026-07-20T19:19:25.726Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62026-07-20T19:19:27.388Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72026-07-20T19:19:27.393Zdownloading input: /input/build/work/bins/crutest.gz
82026-07-20T19:19:28.939Zdownloaded input: /input/build/work/bins/crutest.gz
92026-07-20T19:19:28.945Zdownloading input: /input/build/work/bins/dsc.gz
102026-07-20T19:19:29.638Zdownloaded input: /input/build/work/bins/dsc.gz
112026-07-20T19:19:29.647Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122026-07-20T19:19:29.677Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132026-07-20T19:19:29.686Zdownloading input: /input/build/work/scripts/test_ds.sh
142026-07-20T19:19:29.714Zdownloaded input: /input/build/work/scripts/test_ds.sh
152026-07-20T19:19:29.714Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162026-07-20T19:19:29.761Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172026-07-20T19:19:29.761Zdownloading input: /input/build/work/scripts/test_repair.sh
182026-07-20T19:19:29.798Zdownloaded input: /input/build/work/scripts/test_repair.sh
192026-07-20T19:19:29.808Zdownloading input: /input/build/work/scripts/test_replay.sh
202026-07-20T19:19:29.829Zdownloaded input: /input/build/work/scripts/test_replay.sh
212026-07-20T19:19:29.829Zdownloading input: /input/build/work/scripts/test_up.sh
222026-07-20T19:19:29.871Zdownloaded input: /input/build/work/scripts/test_up.sh
232026-07-20T19:19:29.871Zdownloading input: /input/build/work/scripts/upstairs_info.d
242026-07-20T19:19:29.910Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252026-07-20T19:19:29.917Zdownloading input: /input/build/tmp/cargo-test-out.log
262026-07-20T19:19:30.451Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272026-07-20T19:19:30.451Zstarting task 0: "setup"
282026-07-20T19:19:30.459Z++ uname -s
292026-07-20T19:19:30.459Z+ kern=SunOS
302026-07-20T19:19:30.459Z+ build_user=build
312026-07-20T19:19:30.459Z+ build_uid=12345
322026-07-20T19:19:30.459Z+ work_dir=/work
332026-07-20T19:19:30.459Z+ input_dir=/input
342026-07-20T19:19:30.460Z+ [[ 0 == 12345 ]]
352026-07-20T19:19:30.460Z+ case "$kern" in
362026-07-20T19:19:30.460Z+ groupadd -g 12345 build
372026-07-20T19:19:30.460Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382026-07-20T19:19:32.467Z+ zfs create -o mountpoint=/work rpool/work
392026-07-20T19:19:32.892Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402026-07-20T19:19:32.908Z+ home_fs=zfs
412026-07-20T19:19:32.908Z+ [[ zfs == autofs ]]
422026-07-20T19:19:32.908Z+ mkdir -p /home/build
432026-07-20T19:19:32.908Z+ chown build:build /home/build /work
442026-07-20T19:19:33.838Z+ chmod 0700 /home/build /work
452026-07-20T19:19:33.845Zprocess exited: duration 3388 ms, exit code 0
 
462026-07-20T19:19:33.863Zstarting task 1: "authentication"
472026-07-20T19:19:33.905Zprocess exited: duration 37 ms, exit code 0
 
482026-07-20T19:19:33.936Zstarting task 2: "build"
492026-07-20T19:19:33.943Z+ banner cores
502026-07-20T19:19:33.943Z
512026-07-20T19:19:33.943Z #### #### ##### ###### ####
522026-07-20T19:19:33.943Z # # # # # # # #
532026-07-20T19:19:33.943Z # # # # # ##### ####
542026-07-20T19:19:33.943Z # # # ##### # #
552026-07-20T19:19:33.944Z # # # # # # # # #
562026-07-20T19:19:33.944Z #### #### # # ###### ####
572026-07-20T19:19:33.944Z
582026-07-20T19:19:33.944Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
592026-07-20T19:19:33.957Z+ banner unpack
602026-07-20T19:19:33.958Z
612026-07-20T19:19:33.958Z # # # # ##### ## #### # #
622026-07-20T19:19:33.958Z # # ## # # # # # # # # #
632026-07-20T19:19:33.958Z # # # # # # # # # # ####
642026-07-20T19:19:33.958Z # # # # # ##### ###### # # #
652026-07-20T19:19:33.958Z # # # ## # # # # # # #
662026-07-20T19:19:33.958Z #### # # # # # #### # #
672026-07-20T19:19:33.958Z
682026-07-20T19:19:33.958Z+ mkdir -p /var/tmp/bins
692026-07-20T19:19:33.958Z+ for t in "$input/bins/"*.gz
702026-07-20T19:19:33.958Z++ basename /input/build/work/bins/crucible-downstairs.gz
712026-07-20T19:19:33.958Z+ b=crucible-downstairs.gz
722026-07-20T19:19:33.958Z+ b=crucible-downstairs
732026-07-20T19:19:33.958Z+ gunzip
742026-07-20T19:19:36.524Z+ chmod +x /var/tmp/bins/crucible-downstairs
752026-07-20T19:19:36.529Z+ for t in "$input/bins/"*.gz
762026-07-20T19:19:36.529Z++ basename /input/build/work/bins/crucible-hammer.gz
772026-07-20T19:19:36.529Z+ b=crucible-hammer.gz
782026-07-20T19:19:36.529Z+ b=crucible-hammer
792026-07-20T19:19:36.529Z+ gunzip
802026-07-20T19:19:38.944Z+ chmod +x /var/tmp/bins/crucible-hammer
812026-07-20T19:19:38.950Z+ for t in "$input/bins/"*.gz
822026-07-20T19:19:38.950Z++ basename /input/build/work/bins/crutest.gz
832026-07-20T19:19:38.950Z+ b=crutest.gz
842026-07-20T19:19:38.950Z+ b=crutest
852026-07-20T19:19:38.950Z+ gunzip
862026-07-20T19:19:41.495Z+ chmod +x /var/tmp/bins/crutest
872026-07-20T19:19:41.505Z+ for t in "$input/bins/"*.gz
882026-07-20T19:19:41.505Z++ basename /input/build/work/bins/dsc.gz
892026-07-20T19:19:41.505Z+ b=dsc.gz
902026-07-20T19:19:41.505Z+ b=dsc
912026-07-20T19:19:41.505Z+ gunzip
922026-07-20T19:19:42.627Z+ chmod +x /var/tmp/bins/dsc
932026-07-20T19:19:42.632Z+ export BINDIR=/var/tmp/bins
942026-07-20T19:19:42.633Z+ BINDIR=/var/tmp/bins
952026-07-20T19:19:42.633Z+ jobpid=1090
962026-07-20T19:19:42.633Z+ echo 'Setup debug logging'
972026-07-20T19:19:42.633ZSetup debug logging
982026-07-20T19:19:42.633Z+ mkdir /tmp/debug
992026-07-20T19:19:42.633Z+ sleep 3600
1002026-07-20T19:19:42.633Z+ psrinfo -v
1012026-07-20T19:19:42.633Z+ df -h
1022026-07-20T19:19:42.638Z+ prstat -d d -mLc 1
1032026-07-20T19:19:42.638Z+ iostat -T d -xn 1
1042026-07-20T19:19:42.638Z+ mpstat -T d 1
1052026-07-20T19:19:42.638Z+ vmstat -T d -p 1
1062026-07-20T19:19:42.641Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1072026-07-20T19:19:42.641Z+ banner test_up_2r_encrypted
1082026-07-20T19:19:42.641Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1092026-07-20T19:19:42.642Z #####
1102026-07-20T19:19:42.642Z ##### ###### #### ##### # # ##### # # #####
1112026-07-20T19:19:42.642Z # # # # # # # # # # #
1122026-07-20T19:19:42.642Z # ##### #### # # # # # ##### # #
1132026-07-20T19:19:42.642Z # # # # # # ##### # #####
1142026-07-20T19:19:42.642Z # # # # # # # # # # #
1152026-07-20T19:19:42.642Z # ###### #### # ####### #### # ####### ####### # #
1162026-07-20T19:19:42.642Z
1172026-07-20T19:19:42.642Z+ ptime -m bash /input/build/work/scripts/test_up.sh -r 2 -N encrypted
1182026-07-20T19:19:42.649Z/input/build/work
1192026-07-20T19:19:42.663ZUsing 2 region sets
1202026-07-20T19:19:42.663ZTurn off color for downstairs dump
1212026-07-20T19:19:42.668ZUpstairs using key: 3bhewsXubb/rg3nrT9tV9tZL/IMW+0TwcWsCN+SmaKY=
1222026-07-20T19:19:42.675Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1232026-07-20T19:19:42.675ZCreating 6 downstairs regions
1242026-07-20T19:19:42.920ZStarting 6 downstairs
1252026-07-20T19:19:42.925Zdsc started at PID: 1138
1262026-07-20T19:19:48.010ZDisable automatic restart on all downstairs
1272026-07-20T19:19:48.023Z
1282026-07-20T19:19:48.023ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1292026-07-20T19:19:48.026ZRunning test: span
1302026-07-20T19:19:52.100ZCompleted test: span
1312026-07-20T19:19:57.104ZRunning test: big
1322026-07-20T19:20:05.022ZCompleted test: big
1332026-07-20T19:20:06.417ZRunning test: dep
1342026-07-20T19:20:26.137ZCompleted test: dep
1352026-07-20T19:20:31.144ZRunning test: balloon
1362026-07-20T19:22:58.519ZCompleted test: balloon
1372026-07-20T19:23:03.525ZRunning test: deactivate
1382026-07-20T19:23:07.868ZCompleted test: deactivate
1392026-07-20T19:23:12.875ZRunning hammer
1402026-07-20T19:23:20.101ZRun repair tests
1412026-07-20T19:23:20.105Z/var/tmp/bins/crutest fill -g 31 -q --key 3bhewsXubb/rg3nrT9tV9tZL/IMW+0TwcWsCN+SmaKY= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1422026-07-20T19:23:20.121Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.358077132Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"task":"crutest"}
1432026-07-20T19:23:20.126Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.358260138Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"task":"crutest"}
1442026-07-20T19:23:20.126Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.359052618Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"task":"crutest"}
1452026-07-20T19:23:20.127Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.3592828Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181}
1462026-07-20T19:23:20.127Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3ecc16dae322dc64f81e40811af0ed148a9fbca0\",\n git_commit_timestamp: \"2026-07-20T19:05:26.000000000Z\",\n git_branch: \"alan/printext\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.359311258Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181}
1472026-07-20T19:23:20.127Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.359320597Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181}
1482026-07-20T19:23:20.127Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.359333236Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181}
1492026-07-20T19:23:20.127Z{"msg":"Crucible c4bb88ba-f0c6-47b7-a617-c2ba2c8ba116 has session id: ea2bbab3-2323-4fe2-94c0-3cc7243081cc","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.359716857Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1502026-07-20T19:23:20.127Z{"msg":"Upstairs opts: Upstairs UUID: c4bb88ba-f0c6-47b7-a617-c2ba2c8ba116, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.359741955Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1512026-07-20T19:23:20.127Z{"msg":"Crucible stats registered with UUID: c4bb88ba-f0c6-47b7-a617-c2ba2c8ba116","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.359752804Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1522026-07-20T19:23:20.128Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-20T19:23:18.359762024Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1532026-07-20T19:23:20.128Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.359890254Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"io task","client":"0","":"downstairs","session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1542026-07-20T19:23:20.128Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.359925481Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"io task","client":"1","":"downstairs","session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1552026-07-20T19:23:20.128Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.35994184Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"io task","client":"2","":"downstairs","session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1562026-07-20T19:23:20.128Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.36073052Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"task":"crutest"}
1572026-07-20T19:23:20.128Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.360792355Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181}
1582026-07-20T19:23:20.128Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3ecc16dae322dc64f81e40811af0ed148a9fbca0\",\n git_commit_timestamp: \"2026-07-20T19:05:26.000000000Z\",\n git_branch: \"alan/printext\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.360813273Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181}
1592026-07-20T19:23:20.128Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.360822233Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181}
1602026-07-20T19:23:20.128Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.360834292Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181}
1612026-07-20T19:23:20.128Z{"msg":"Crucible c4bb88ba-f0c6-47b7-a617-c2ba2c8ba116 has session id: 38a919cb-dfb3-48fe-afc3-5f91668f6020","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.361014918Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
1622026-07-20T19:23:20.128Z{"msg":"Upstairs opts: Upstairs UUID: c4bb88ba-f0c6-47b7-a617-c2ba2c8ba116, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.361037636Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
1632026-07-20T19:23:20.128Z{"msg":"Crucible stats registered with UUID: c4bb88ba-f0c6-47b7-a617-c2ba2c8ba116","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.361056385Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
1642026-07-20T19:23:20.128Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-20T19:23:18.361065444Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
1652026-07-20T19:23:20.129Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.361075403Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"io task","client":"1","":"downstairs","session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
1662026-07-20T19:23:20.129Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.361086762Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"io task","client":"2","":"downstairs","session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
1672026-07-20T19:23:20.129Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.361097742Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"io task","client":"0","":"downstairs","session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
1682026-07-20T19:23:20.129Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.361184325Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"task":"crutest"}
1692026-07-20T19:23:20.129Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.361197804Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181}
1702026-07-20T19:23:20.129Z{"msg":"c4bb88ba-f0c6-47b7-a617-c2ba2c8ba116 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.361216103Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1712026-07-20T19:23:20.129Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.361225752Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"client":"0","":"downstairs","session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1722026-07-20T19:23:20.129Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.361235781Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"client":"1","":"downstairs","session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1732026-07-20T19:23:20.129Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.36124565Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"client":"2","":"downstairs","session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1742026-07-20T19:23:20.129Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.361257699Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"io task","client":"0","":"downstairs","session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1752026-07-20T19:23:20.129Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.361268469Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"io task","client":"2","":"downstairs","session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1762026-07-20T19:23:20.129Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.361302136Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"io task","client":"1","":"downstairs","session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1772026-07-20T19:23:20.129Z{"msg":"ds_connection connected from Ok(127.0.0.1:61115)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.361325094Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"io task","client":"0","":"downstairs","session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1782026-07-20T19:23:20.130Z{"msg":"ds_connection connected from Ok(127.0.0.1:63580)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.361335793Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"io task","client":"1","":"downstairs","session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1792026-07-20T19:23:20.130Z{"msg":"ds_connection connected from Ok(127.0.0.1:43975)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.361391759Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"io task","client":"2","":"downstairs","session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1802026-07-20T19:23:20.130Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.361933838Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"client":"1","":"downstairs","session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1812026-07-20T19:23:20.130Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.36203023Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"client":"0","":"downstairs","session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1822026-07-20T19:23:20.130Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362047989Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"client":"2","":"downstairs","session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1832026-07-20T19:23:20.130Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362139312Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1842026-07-20T19:23:20.134Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362208687Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1852026-07-20T19:23:20.134Z{"msg":"[0]R flush_numbers: [3996, 3996, 3996, 3996, 3996]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362255383Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"downstairs","":"downstairs","session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1862026-07-20T19:23:20.134Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362266382Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"downstairs","":"downstairs","session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1872026-07-20T19:23:20.134Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362276892Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"downstairs","":"downstairs","session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1882026-07-20T19:23:20.134Z{"msg":"[1]R flush_numbers: [3996, 3996, 3996, 3996, 3996]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362287131Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"downstairs","":"downstairs","session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1892026-07-20T19:23:20.134Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.36229729Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"downstairs","":"downstairs","session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1902026-07-20T19:23:20.134Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362307899Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"downstairs","":"downstairs","session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1912026-07-20T19:23:20.134Z{"msg":"[2]R flush_numbers: [3996, 3996, 3996, 3996, 3996]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362318198Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"downstairs","":"downstairs","session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1922026-07-20T19:23:20.134Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362328498Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"downstairs","":"downstairs","session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1932026-07-20T19:23:20.134Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362338717Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"downstairs","":"downstairs","session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1942026-07-20T19:23:20.134Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362348136Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"downstairs","":"downstairs","session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1952026-07-20T19:23:20.134Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362358515Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"downstairs","":"downstairs","session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1962026-07-20T19:23:20.134Z{"msg":"Next flush: 3997","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362373824Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"downstairs","":"downstairs","session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1972026-07-20T19:23:20.135Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362384223Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"downstairs","":"downstairs","session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1982026-07-20T19:23:20.135Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362394483Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
1992026-07-20T19:23:20.135Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362403972Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"client":"0","":"downstairs","session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
2002026-07-20T19:23:20.135Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362414191Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"client":"1","":"downstairs","session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
2012026-07-20T19:23:20.135Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.36242468Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"client":"2","":"downstairs","session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
2022026-07-20T19:23:20.135Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.3624349Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
2032026-07-20T19:23:20.135Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362444429Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
2042026-07-20T19:23:20.135Z{"msg":"c4bb88ba-f0c6-47b7-a617-c2ba2c8ba116 is now active with session: ea2bbab3-2323-4fe2-94c0-3cc7243081cc","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362455128Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
2052026-07-20T19:23:20.135Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362468777Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"session_id":"ea2bbab3-2323-4fe2-94c0-3cc7243081cc"}
2062026-07-20T19:23:20.135Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362478496Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181}
2072026-07-20T19:23:20.135Z{"msg":"Activated sub_volume c4bb88ba-f0c6-47b7-a617-c2ba2c8ba116","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362487216Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181}
2082026-07-20T19:23:20.135Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362495675Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181}
2092026-07-20T19:23:20.135Z{"msg":"c4bb88ba-f0c6-47b7-a617-c2ba2c8ba116 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362504254Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2102026-07-20T19:23:20.135Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362513684Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"client":"0","":"downstairs","session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2112026-07-20T19:23:20.135Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362523623Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"client":"1","":"downstairs","session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2122026-07-20T19:23:20.135Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362537342Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"client":"2","":"downstairs","session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2132026-07-20T19:23:20.135Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362547431Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"io task","client":"2","":"downstairs","session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2142026-07-20T19:23:20.135Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.36256173Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"io task","client":"0","":"downstairs","session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2152026-07-20T19:23:20.135Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362606356Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"io task","client":"1","":"downstairs","session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2162026-07-20T19:23:20.135Z{"msg":"ds_connection connected from Ok(127.0.0.1:57292)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362636754Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"io task","client":"2","":"downstairs","session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2172026-07-20T19:23:20.135Z{"msg":"ds_connection connected from Ok(127.0.0.1:63243)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362664622Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"io task","client":"1","":"downstairs","session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2182026-07-20T19:23:20.135Z{"msg":"ds_connection connected from Ok(127.0.0.1:36771)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362674971Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"io task","client":"0","":"downstairs","session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2192026-07-20T19:23:20.135Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362884665Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"client":"0","":"downstairs","session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2202026-07-20T19:23:20.135Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.362934961Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"client":"1","":"downstairs","session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2212026-07-20T19:23:20.135Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.363011266Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2222026-07-20T19:23:20.135Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.363022595Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"client":"2","":"downstairs","session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2232026-07-20T19:23:20.135Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.363048823Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2242026-07-20T19:23:20.135Z{"msg":"[0]R flush_numbers: [5314, 5315, 5313, 5315, 5315]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.363133166Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"downstairs","":"downstairs","session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2252026-07-20T19:23:20.135Z{"msg":"[0]R generation: [14, 14, 13, 14, 14]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.363144195Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"downstairs","":"downstairs","session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2262026-07-20T19:23:20.135Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.363154655Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"downstairs","":"downstairs","session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2272026-07-20T19:23:20.136Z{"msg":"[1]R flush_numbers: [5314, 5315, 5313, 5315, 5315]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.363165184Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"downstairs","":"downstairs","session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2282026-07-20T19:23:20.136Z{"msg":"[1]R generation: [14, 14, 13, 14, 14]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.363175173Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"downstairs","":"downstairs","session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2292026-07-20T19:23:20.136Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.363185752Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"downstairs","":"downstairs","session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2302026-07-20T19:23:20.136Z{"msg":"[2]R flush_numbers: [5314, 5315, 5313, 5315, 5315]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.363196111Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"downstairs","":"downstairs","session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2312026-07-20T19:23:20.136Z{"msg":"[2]R generation: [14, 14, 13, 14, 14]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.363206201Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"downstairs","":"downstairs","session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2322026-07-20T19:23:20.136Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.36321666Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"downstairs","":"downstairs","session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2332026-07-20T19:23:20.136Z{"msg":"Max found gen is 15","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.363226519Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"downstairs","":"downstairs","session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2342026-07-20T19:23:20.136Z{"msg":"Generation requested: 31 >= found:15","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.363236808Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"downstairs","":"downstairs","session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2352026-07-20T19:23:20.136Z{"msg":"Next flush: 5316","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.363246588Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"downstairs","":"downstairs","session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2362026-07-20T19:23:20.136Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.363256537Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"":"downstairs","":"downstairs","session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2372026-07-20T19:23:20.136Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.363266856Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2382026-07-20T19:23:20.136Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.363276045Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"client":"0","":"downstairs","session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2392026-07-20T19:23:20.136Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.363286045Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"client":"1","":"downstairs","session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2402026-07-20T19:23:20.136Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.363295954Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"client":"2","":"downstairs","session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2412026-07-20T19:23:20.136Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.363306473Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2422026-07-20T19:23:20.136Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.363316412Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2432026-07-20T19:23:20.136Z{"msg":"c4bb88ba-f0c6-47b7-a617-c2ba2c8ba116 is now active with session: 38a919cb-dfb3-48fe-afc3-5f91668f6020","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.363326971Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2442026-07-20T19:23:20.137Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.36334851Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"session_id":"38a919cb-dfb3-48fe-afc3-5f91668f6020"}
2452026-07-20T19:23:20.137Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.363358689Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181}
2462026-07-20T19:23:20.137Z{"msg":"Activated sub_volume c4bb88ba-f0c6-47b7-a617-c2ba2c8ba116","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.363367188Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181}
2472026-07-20T19:23:20.137Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.363375798Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"task":"crutest"}
2482026-07-20T19:23:20.137ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
2492026-07-20T19:23:20.137Z{"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.363433993Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"task":"crutest"}
2502026-07-20T19:23:20.151ZRead and Verify all blocks (0..100 range:false)
2512026-07-20T19:23:20.200Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30----------------------------------------------------------------
2522026-07-20T19:23:20.200Z,"time":"2026-07-20T19:23:18.436895566Z","hostname":" Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2532026-07-20T19:23:20.200Zw-01KY0FE5VTQK6S1ZS8T3K93BSW"States:,"pid": Active1181 Active Active
2542026-07-20T19:23:20.200Z,"JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2552026-07-20T19:23:20.200Ztask":"crutest"}
2562026-07-20T19:23:20.200Z 1002 Acked Read 50{ Done" Donemsg" Done:" false
2572026-07-20T19:23:20.200ZCLIENT: Tests done. All submitted work has been ACK'd","v":0," STATES DS:0 DS:1 DS:2 TOTAL
2582026-07-20T19:23:20.203Zname Sent ":" 0 crucible" 0 ," 0 0
2592026-07-20T19:23:20.204Zlevel": Done 30 1 1 1 3
2602026-07-20T19:23:20.204Z Skipped 0 0 0 0
2612026-07-20T19:23:20.204Z," Error time": 0 " 0 0 0
2622026-07-20T19:23:20.204Z2026-07-20T19:23:18.436930713Z"Last Flush: ,"Some(JobId(1001)) hostnameSome(JobId(1001)) "Some(JobId(1001)) :
2632026-07-20T19:23:20.204Z"Downstairs last five completed:w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181 1001 1000
2642026-07-20T19:23:20.204Z,"task":"crutest"}Upstairs last five completed:
2652026-07-20T19:23:20.204Z 1002 1001 1000
2662026-07-20T19:23:20.204Z----------------------------------------------------------------
2672026-07-20T19:23:20.204Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2682026-07-20T19:23:20.204ZStates: Active Active Active
2692026-07-20T19:23:20.204ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2702026-07-20T19:23:20.204Z 1002 Acked Read 50 Done Done Done false
2712026-07-20T19:23:20.204Z STATES DS:0 DS:1 DS:2 TOTAL
2722026-07-20T19:23:20.204Z Sent 0 0 0 0
2732026-07-20T19:23:20.204Z Done 1 1 1 3
2742026-07-20T19:23:20.204Z Skipped 0 0 0 0
2752026-07-20T19:23:20.204Z Error 0 0 0 0
2762026-07-20T19:23:20.204ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2772026-07-20T19:23:20.204ZDownstairs last five completed: 1001 1000
2782026-07-20T19:23:20.204ZUpstairs last five completed: 1002 1001 1000
2792026-07-20T19:23:20.204Z{"msg":"CLIENT: Up:0 ds:2 act:6","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:18.437013297Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"task":"crutest"}
2802026-07-20T19:23:24.201Z----------------------------------------------------------------
2812026-07-20T19:23:24.201Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2822026-07-20T19:23:24.201ZDownstairs last five completed: 1003 1002 1001 1000
2832026-07-20T19:23:24.201ZUpstairs last five completed: 1003 1002 1001 1000
2842026-07-20T19:23:24.205Z----------------------------------------------------------------
2852026-07-20T19:23:24.205Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2862026-07-20T19:23:24.205ZDownstairs last five completed: 1003 1002 1001 1000
2872026-07-20T19:23:24.205ZUpstairs last five completed: 1003 1002 1001 1000
2882026-07-20T19:23:24.205Z{"msg":"CLIENT: Up:0 ds:0 act:6","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.438161721Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"task":"crutest"}
2892026-07-20T19:23:24.205Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.438215517Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181,"task":"crutest"}
2902026-07-20T19:23:24.205Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-20T19:23:22.438231166Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1181}
2912026-07-20T19:23:24.210ZRepair setup passed
2922026-07-20T19:23:24.213ZCopy the region for /var/tmp/test_up-build/8830
2932026-07-20T19:23:24.213Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2942026-07-20T19:23:24.216Z/var/tmp/bins/crutest fill -g 32 -q --key 3bhewsXubb/rg3nrT9tV9tZL/IMW+0TwcWsCN+SmaKY= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2952026-07-20T19:23:24.233Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.469679336Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"task":"crutest"}
2962026-07-20T19:23:24.238Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.469993502Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"task":"crutest"}
2972026-07-20T19:23:24.238Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.470660201Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"task":"crutest"}
2982026-07-20T19:23:24.238Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.470895483Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187}
2992026-07-20T19:23:24.238Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3ecc16dae322dc64f81e40811af0ed148a9fbca0\",\n git_commit_timestamp: \"2026-07-20T19:05:26.000000000Z\",\n git_branch: \"alan/printext\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.470921091Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187}
3002026-07-20T19:23:24.239Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.47093423Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187}
3012026-07-20T19:23:24.239Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.470946579Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187}
3022026-07-20T19:23:24.239Z{"msg":"Crucible f7a8bdb6-23d4-48b9-87f7-0f3bc0d7e755 has session id: 9f2925b2-2d04-462b-a970-b013fb283365","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.471399494Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3032026-07-20T19:23:24.239Z{"msg":"Upstairs opts: Upstairs UUID: f7a8bdb6-23d4-48b9-87f7-0f3bc0d7e755, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.471425682Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3042026-07-20T19:23:24.239Z{"msg":"Crucible stats registered with UUID: f7a8bdb6-23d4-48b9-87f7-0f3bc0d7e755","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.471435512Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3052026-07-20T19:23:24.239Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-20T19:23:22.471449041Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3062026-07-20T19:23:24.239Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.47158123Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"io task","client":"1","":"downstairs","session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3072026-07-20T19:23:24.239Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.471603259Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"io task","client":"0","":"downstairs","session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3082026-07-20T19:23:24.239Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.471613568Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"io task","client":"2","":"downstairs","session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3092026-07-20T19:23:24.239Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.472526748Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"task":"crutest"}
3102026-07-20T19:23:24.239Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.472589353Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187}
3112026-07-20T19:23:24.239Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3ecc16dae322dc64f81e40811af0ed148a9fbca0\",\n git_commit_timestamp: \"2026-07-20T19:05:26.000000000Z\",\n git_branch: \"alan/printext\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.472612882Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187}
3122026-07-20T19:23:24.239Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.472625151Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187}
3132026-07-20T19:23:24.239Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.47263681Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187}
3142026-07-20T19:23:24.239Z{"msg":"Crucible f7a8bdb6-23d4-48b9-87f7-0f3bc0d7e755 has session id: 56135274-1efc-4734-87c4-b5138b111d3b","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.472905609Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3152026-07-20T19:23:24.239Z{"msg":"Upstairs opts: Upstairs UUID: f7a8bdb6-23d4-48b9-87f7-0f3bc0d7e755, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.472924438Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3162026-07-20T19:23:24.239Z{"msg":"Crucible stats registered with UUID: f7a8bdb6-23d4-48b9-87f7-0f3bc0d7e755","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.472934487Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3172026-07-20T19:23:24.239Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-20T19:23:22.472951366Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3182026-07-20T19:23:24.239Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.472998142Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"io task","client":"0","":"downstairs","session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3192026-07-20T19:23:24.239Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.473013881Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"io task","client":"1","":"downstairs","session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3202026-07-20T19:23:24.239Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.47302394Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"io task","client":"2","":"downstairs","session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3212026-07-20T19:23:24.240Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.473137082Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"task":"crutest"}
3222026-07-20T19:23:24.240Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.473146761Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187}
3232026-07-20T19:23:24.246Z{"msg":"f7a8bdb6-23d4-48b9-87f7-0f3bc0d7e755 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.473199797Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3242026-07-20T19:23:24.246Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.473213796Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"client":"0","":"downstairs","session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3252026-07-20T19:23:24.246Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.473242464Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"client":"1","":"downstairs","session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3262026-07-20T19:23:24.246Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.473271071Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"client":"2","":"downstairs","session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3272026-07-20T19:23:24.246Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.473281741Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"io task","client":"2","":"downstairs","session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3282026-07-20T19:23:24.246Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.47329657Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"io task","client":"0","":"downstairs","session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3292026-07-20T19:23:24.246Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.473307379Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"io task","client":"1","":"downstairs","session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3302026-07-20T19:23:24.246Z{"msg":"ds_connection connected from Ok(127.0.0.1:58555)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.473318388Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"io task","client":"0","":"downstairs","session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3312026-07-20T19:23:24.246Z{"msg":"ds_connection connected from Ok(127.0.0.1:64270)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.473329217Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"io task","client":"2","":"downstairs","session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3322026-07-20T19:23:24.246Z{"msg":"ds_connection connected from Ok(127.0.0.1:61299)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.473343766Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"io task","client":"1","":"downstairs","session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3332026-07-20T19:23:24.246Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.4740752Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"client":"1","":"downstairs","session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3342026-07-20T19:23:24.246Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.474162423Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"client":"0","":"downstairs","session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3352026-07-20T19:23:24.247Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.474178872Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"client":"2","":"downstairs","session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3362026-07-20T19:23:24.247Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.474329391Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3372026-07-20T19:23:24.247Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.474366908Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3382026-07-20T19:23:24.247Z{"msg":"[0]R flush_numbers: [3997, 3997, 3997, 3997, 3997]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.474396716Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"downstairs","":"downstairs","session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3392026-07-20T19:23:24.247Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.474411334Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"downstairs","":"downstairs","session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3402026-07-20T19:23:24.247Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.474421704Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"downstairs","":"downstairs","session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3412026-07-20T19:23:24.247Z{"msg":"[1]R flush_numbers: [3997, 3997, 3997, 3997, 3997]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.474436243Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"downstairs","":"downstairs","session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3422026-07-20T19:23:24.247Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.474446322Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"downstairs","":"downstairs","session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3432026-07-20T19:23:24.247Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.474460561Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"downstairs","":"downstairs","session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3442026-07-20T19:23:24.247Z{"msg":"[2]R flush_numbers: [3997, 3997, 3997, 3997, 3997]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.47447064Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"downstairs","":"downstairs","session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3452026-07-20T19:23:24.247Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.474484579Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"downstairs","":"downstairs","session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3462026-07-20T19:23:24.247Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.474494838Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"downstairs","":"downstairs","session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3472026-07-20T19:23:24.247Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.474508347Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"downstairs","":"downstairs","session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3482026-07-20T19:23:24.247Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.474518076Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"downstairs","":"downstairs","session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3492026-07-20T19:23:24.247Z{"msg":"Next flush: 3998","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.474531395Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"downstairs","":"downstairs","session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3502026-07-20T19:23:24.247Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.474541165Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"downstairs","":"downstairs","session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3512026-07-20T19:23:24.247Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.474555523Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3522026-07-20T19:23:24.247Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.474564363Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"client":"0","":"downstairs","session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3532026-07-20T19:23:24.247Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.474578202Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"client":"1","":"downstairs","session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3542026-07-20T19:23:24.247Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.474588491Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"client":"2","":"downstairs","session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3552026-07-20T19:23:24.247Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.47459892Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3562026-07-20T19:23:24.247Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.474608699Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3572026-07-20T19:23:24.248Z{"msg":"f7a8bdb6-23d4-48b9-87f7-0f3bc0d7e755 is now active with session: 9f2925b2-2d04-462b-a970-b013fb283365","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.474619289Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3582026-07-20T19:23:24.248Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.474632578Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"session_id":"9f2925b2-2d04-462b-a970-b013fb283365"}
3592026-07-20T19:23:24.248Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.474641907Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187}
3602026-07-20T19:23:24.248Z{"msg":"Activated sub_volume f7a8bdb6-23d4-48b9-87f7-0f3bc0d7e755","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.474650346Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187}
3612026-07-20T19:23:24.248Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.474662715Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187}
3622026-07-20T19:23:24.248Z{"msg":"f7a8bdb6-23d4-48b9-87f7-0f3bc0d7e755 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.474670705Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3632026-07-20T19:23:24.248Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.474679804Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"client":"0","":"downstairs","session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3642026-07-20T19:23:24.248Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.474770147Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"client":"1","":"downstairs","session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3652026-07-20T19:23:24.249Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.474815374Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"client":"2","":"downstairs","session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3662026-07-20T19:23:24.249Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.474831312Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"io task","client":"2","":"downstairs","session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3672026-07-20T19:23:24.249Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.474846121Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"io task","client":"0","":"downstairs","session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3682026-07-20T19:23:24.249Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.47485606Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"io task","client":"1","":"downstairs","session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3692026-07-20T19:23:24.249Z{"msg":"ds_connection connected from Ok(127.0.0.1:57534)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.47486657Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"io task","client":"2","":"downstairs","session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3702026-07-20T19:23:24.249Z{"msg":"ds_connection connected from Ok(127.0.0.1:55368)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.474884548Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"io task","client":"1","":"downstairs","session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3712026-07-20T19:23:24.249Z{"msg":"ds_connection connected from Ok(127.0.0.1:56966)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.474894748Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"io task","client":"0","":"downstairs","session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3722026-07-20T19:23:24.249Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.475046726Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"client":"0","":"downstairs","session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3732026-07-20T19:23:24.249Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.475084993Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"client":"1","":"downstairs","session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3742026-07-20T19:23:24.249Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.475187635Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"client":"2","":"downstairs","session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3752026-07-20T19:23:24.249Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.475233612Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3762026-07-20T19:23:24.249Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.47525093Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3772026-07-20T19:23:24.249Z{"msg":"[0]R flush_numbers: [5316, 5316, 5316, 5316, 5316]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.475330594Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"downstairs","":"downstairs","session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3782026-07-20T19:23:24.249Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.475345243Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"downstairs","":"downstairs","session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3792026-07-20T19:23:24.249Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.475355012Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"downstairs","":"downstairs","session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3802026-07-20T19:23:24.249Z{"msg":"[1]R flush_numbers: [5316, 5316, 5316, 5316, 5316]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.475369101Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"downstairs","":"downstairs","session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3812026-07-20T19:23:24.250Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.475378551Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"downstairs","":"downstairs","session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3822026-07-20T19:23:24.250Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.47539266Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"downstairs","":"downstairs","session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3832026-07-20T19:23:24.250Z{"msg":"[2]R flush_numbers: [5316, 5316, 5316, 5316, 5316]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.475402419Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"downstairs","":"downstairs","session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3842026-07-20T19:23:24.250Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.475416388Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"downstairs","":"downstairs","session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3852026-07-20T19:23:24.250Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.475426147Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"downstairs","":"downstairs","session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3862026-07-20T19:23:24.250Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.475435926Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"downstairs","":"downstairs","session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3872026-07-20T19:23:24.250Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.475450075Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"downstairs","":"downstairs","session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3882026-07-20T19:23:24.250Z{"msg":"Next flush: 5317","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.475459104Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"downstairs","":"downstairs","session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3892026-07-20T19:23:24.250Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.475472913Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"":"downstairs","":"downstairs","session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3902026-07-20T19:23:24.250Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.475482603Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3912026-07-20T19:23:24.250Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.475495652Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"client":"0","":"downstairs","session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3922026-07-20T19:23:24.250Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.475505251Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"client":"1","":"downstairs","session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3932026-07-20T19:23:24.250Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.47551893Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"client":"2","":"downstairs","session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3942026-07-20T19:23:24.250Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.475528719Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3952026-07-20T19:23:24.250Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.475541878Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3962026-07-20T19:23:24.250Z{"msg":"f7a8bdb6-23d4-48b9-87f7-0f3bc0d7e755 is now active with session: 56135274-1efc-4734-87c4-b5138b111d3b","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.475551597Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3972026-07-20T19:23:24.250Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.475564566Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"session_id":"56135274-1efc-4734-87c4-b5138b111d3b"}
3982026-07-20T19:23:24.250Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.475573416Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187}
3992026-07-20T19:23:24.251Z{"msg":"Activated sub_volume f7a8bdb6-23d4-48b9-87f7-0f3bc0d7e755","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.475585675Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187}
4002026-07-20T19:23:24.251Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.475594094Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"task":"crutest"}
4012026-07-20T19:23:24.251ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
4022026-07-20T19:23:24.251ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4032026-07-20T19:23:24.251Z{"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.476052429Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"task":"crutest"}
4042026-07-20T19:23:24.264ZRead and Verify all blocks (0..100 range:false)
4052026-07-20T19:23:24.311Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30----------------------------------------------------------------
4062026-07-20T19:23:24.311Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4072026-07-20T19:23:24.311Z,States:"time Active" Active:" Active
4082026-07-20T19:23:24.311Z2026-07-20T19:23:22.547815131Z"JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4092026-07-20T19:23:24.311Z,"hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid": 1002 Acked Read 501187 Done Done Done false,
4102026-07-20T19:23:24.311Z"task":"crutest"} STATES DS:0 DS:1 DS:2 TOTAL
4112026-07-20T19:23:24.311Z
4122026-07-20T19:23:24.311Z Sent 0 0 0 0
4132026-07-20T19:23:24.311Z{ Done 1 " 1 1 msg" 3
4142026-07-20T19:23:24.311Z:" Skipped 0 0 0 0
4152026-07-20T19:23:24.311ZCLIENT: Tests done. All submitted work has been ACK'd"," Error v" 0 : 0 0 0 , 0
4162026-07-20T19:23:24.311Z"nameLast Flush: ":"crucible"Some(JobId(1001)) ,"Some(JobId(1001)) level"Some(JobId(1001)) :
4172026-07-20T19:23:24.311Z30Downstairs last five completed: 1001 1000
4182026-07-20T19:23:24.311Z,"time":"Upstairs last five completed: 1002 10012026-07-20T19:23:22.547854958Z" 1000,"
4192026-07-20T19:23:24.312Zhostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"task":"crutest"}
4202026-07-20T19:23:24.316Z----------------------------------------------------------------
4212026-07-20T19:23:24.316Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4222026-07-20T19:23:24.316ZStates: Active Active Active
4232026-07-20T19:23:24.316ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4242026-07-20T19:23:24.316Z 1002 Acked Read 50 Done Done Done false
4252026-07-20T19:23:24.316Z STATES DS:0 DS:1 DS:2 TOTAL
4262026-07-20T19:23:24.316Z Sent 0 0 0 0
4272026-07-20T19:23:24.316Z Done 1 1 1 3
4282026-07-20T19:23:24.316Z Skipped 0 0 0 0
4292026-07-20T19:23:24.316Z Error 0 0 0 0
4302026-07-20T19:23:24.316ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4312026-07-20T19:23:24.316ZDownstairs last five completed: 1001 1000
4322026-07-20T19:23:24.316ZUpstairs last five completed: 1002 1001 1000
4332026-07-20T19:23:24.316Z{"msg":"CLIENT: Up:0 ds:2 act:6","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:22.547926813Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"task":"crutest"}
4342026-07-20T19:23:28.324Z----------------------------------------------------------------
4352026-07-20T19:23:28.324Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4362026-07-20T19:23:28.324ZDownstairs last five completed: 1003 1002 1001 1000
4372026-07-20T19:23:28.324ZUpstairs last five completed: 1003 1002 1001 1000
4382026-07-20T19:23:28.324Z----------------------------------------------------------------
4392026-07-20T19:23:28.324Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4402026-07-20T19:23:28.325ZDownstairs last five completed: 1003 1002 1001 1000
4412026-07-20T19:23:28.325ZUpstairs last five completed: 1003 1002 1001 1000
4422026-07-20T19:23:28.333Z{"msg":"CLIENT: Up:0 ds:0 act:6","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.549982189Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"task":"crutest"}
4432026-07-20T19:23:28.333Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.550053784Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187,"task":"crutest"}
4442026-07-20T19:23:28.333Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-20T19:23:26.550073592Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1187}
4452026-07-20T19:23:28.334ZRepair part 1 passed
4462026-07-20T19:23:28.334Z
4472026-07-20T19:23:28.334ZKill the current downstairs
4482026-07-20T19:23:28.366ZDownstairs 2 stopped
4492026-07-20T19:23:28.366Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
4502026-07-20T19:23:28.373ZNow put back the original so we have a mismatch
4512026-07-20T19:23:28.373Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
4522026-07-20T19:23:28.373ZRestart downstairs with old directory
4532026-07-20T19:23:28.386Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4542026-07-20T19:23:28.400ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4552026-07-20T19:23:28.424Z 0 000-009 32 32 31 3998 3998 3997 F F F <---
4562026-07-20T19:23:28.424Z 1 010-019 32 32 31 3998 3998 3997 F F F <---
4572026-07-20T19:23:28.424Z 2 020-029 32 32 31 3998 3998 3997 F F F <---
4582026-07-20T19:23:28.424Z 3 030-039 32 32 31 3998 3998 3997 F F F <---
4592026-07-20T19:23:28.424Z 4 040-049 32 32 31 3998 3998 3997 F F F <---
4602026-07-20T19:23:28.424ZMax gen: 32, Max flush: 3998
4612026-07-20T19:23:28.424ZError: Difference in extent metadata found!
4622026-07-20T19:23:28.424Zdump test found error as expected
4632026-07-20T19:23:28.424Z
4642026-07-20T19:23:28.424Z
4652026-07-20T19:23:28.424Z/var/tmp/bins/crutest verify --range -g 33 -q --key 3bhewsXubb/rg3nrT9tV9tZL/IMW+0TwcWsCN+SmaKY= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
4662026-07-20T19:23:28.458Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.648449033Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"task":"crutest"}
4672026-07-20T19:23:28.459Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.648703524Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"task":"crutest"}
4682026-07-20T19:23:28.459Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.64927625Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"task":"crutest"}
4692026-07-20T19:23:28.459Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.649511892Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195}
4702026-07-20T19:23:28.459Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3ecc16dae322dc64f81e40811af0ed148a9fbca0\",\n git_commit_timestamp: \"2026-07-20T19:05:26.000000000Z\",\n git_branch: \"alan/printext\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.64954059Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195}
4712026-07-20T19:23:28.459Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.649553969Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195}
4722026-07-20T19:23:28.459Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.649566198Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195}
4732026-07-20T19:23:28.459Z{"msg":"Crucible 06490a9f-e383-470f-9ee1-a5ecdf9627fd has session id: e788d3c7-afab-4baf-a411-78857ade5e60","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.64993153Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
4742026-07-20T19:23:28.459Z{"msg":"Upstairs opts: Upstairs UUID: 06490a9f-e383-470f-9ee1-a5ecdf9627fd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.649955958Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
4752026-07-20T19:23:28.459Z{"msg":"Crucible stats registered with UUID: 06490a9f-e383-470f-9ee1-a5ecdf9627fd","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.649966187Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
4762026-07-20T19:23:28.460Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-20T19:23:26.649976116Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
4772026-07-20T19:23:28.460Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.650130915Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"io task","client":"1","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
4782026-07-20T19:23:28.460Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.650172001Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"io task","client":"0","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
4792026-07-20T19:23:28.460Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.65018776Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"io task","client":"2","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
4802026-07-20T19:23:28.460Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.651325383Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"task":"crutest"}
4812026-07-20T19:23:28.460Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.651383369Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195}
4822026-07-20T19:23:28.460Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3ecc16dae322dc64f81e40811af0ed148a9fbca0\",\n git_commit_timestamp: \"2026-07-20T19:05:26.000000000Z\",\n git_branch: \"alan/printext\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.651408917Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195}
4832026-07-20T19:23:28.460Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.651422126Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195}
4842026-07-20T19:23:28.460Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.651433955Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195}
4852026-07-20T19:23:28.460Z{"msg":"Crucible 06490a9f-e383-470f-9ee1-a5ecdf9627fd has session id: 148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.651585594Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
4862026-07-20T19:23:28.461Z{"msg":"Upstairs opts: Upstairs UUID: 06490a9f-e383-470f-9ee1-a5ecdf9627fd, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.651614541Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
4872026-07-20T19:23:28.462Z{"msg":"Crucible stats registered with UUID: 06490a9f-e383-470f-9ee1-a5ecdf9627fd","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.65163406Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
4882026-07-20T19:23:28.462Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-20T19:23:26.651644239Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
4892026-07-20T19:23:28.463Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.651653688Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"io task","client":"0","":"downstairs","session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
4902026-07-20T19:23:28.463Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.651664867Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"io task","client":"2","":"downstairs","session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
4912026-07-20T19:23:28.463Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.651675527Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"io task","client":"1","":"downstairs","session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
4922026-07-20T19:23:28.463Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.651770639Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"task":"crutest"}
4932026-07-20T19:23:28.463Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.651781599Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195}
4942026-07-20T19:23:28.463Z{"msg":"06490a9f-e383-470f-9ee1-a5ecdf9627fd active request set","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.651806037Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
4952026-07-20T19:23:28.463Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.651815416Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"client":"0","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
4962026-07-20T19:23:28.463Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.651825435Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"client":"1","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
4972026-07-20T19:23:28.465Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.651835474Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"client":"2","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
4982026-07-20T19:23:28.465Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.651850163Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"io task","client":"1","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
4992026-07-20T19:23:28.465Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.651860633Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"io task","client":"2","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5002026-07-20T19:23:28.465Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.651871162Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"io task","client":"0","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5012026-07-20T19:23:28.465Z{"msg":"ds_connection connected from Ok(127.0.0.1:34634)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.651914808Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"io task","client":"0","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5022026-07-20T19:23:28.465Z{"msg":"ds_connection connected from Ok(127.0.0.1:59500)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.651930177Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"io task","client":"1","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5032026-07-20T19:23:28.465Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-07-20T19:23:26.652053358Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"io task","client":"2","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5042026-07-20T19:23:28.465Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-07-20T19:23:26.652111863Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"io task","client":"2","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5052026-07-20T19:23:28.465Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-07-20T19:23:26.652141931Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5062026-07-20T19:23:28.465Z{"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.65215637Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"io task","client":"2","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5072026-07-20T19:23:28.467Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.652167169Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"io task","client":"2","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5082026-07-20T19:23:28.467Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.652538821Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"client":"1","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5092026-07-20T19:23:28.467Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.652567819Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"client":"0","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5102026-07-20T19:23:28.467Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.652687589Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5112026-07-20T19:23:28.467Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:26.652702668Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5122026-07-20T19:23:38.418Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.653808806Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"io task","client":"2","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5132026-07-20T19:23:38.426Z{"msg":"ds_connection connected from Ok(127.0.0.1:32838)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.654034089Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"io task","client":"2","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5142026-07-20T19:23:38.426Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.655121666Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"client":"2","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5152026-07-20T19:23:38.426Z{"msg":"[0]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.655335159Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"downstairs","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5162026-07-20T19:23:38.426Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.655358158Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"downstairs","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5172026-07-20T19:23:38.426Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.655377796Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"downstairs","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5182026-07-20T19:23:38.426Z{"msg":"[1]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.655398294Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"downstairs","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5192026-07-20T19:23:38.426Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.655417073Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"downstairs","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5202026-07-20T19:23:38.426Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.655437142Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"downstairs","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5212026-07-20T19:23:38.426Z{"msg":"[2]R flush_numbers: [3997, 3997, 3997, 3997, 3997]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.65545786Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"downstairs","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5222026-07-20T19:23:38.426Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.655477418Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"downstairs","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5232026-07-20T19:23:38.426Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.655497407Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"downstairs","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5242026-07-20T19:23:38.426Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.655516335Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"downstairs","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5252026-07-20T19:23:38.426Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.655536384Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"downstairs","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5262026-07-20T19:23:38.426Z{"msg":"Next flush: 3999","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.655555602Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"downstairs","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5272026-07-20T19:23:38.426Z{"msg":"extent 0 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3998, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3998, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3997, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.65558332Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"mend","":"downstairs","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5282026-07-20T19:23:38.426Z{"msg":"extent:0 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3998, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3998, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3997, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.655612008Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"mend","":"downstairs","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5292026-07-20T19:23:38.426Z{"msg":"found dest for source 0 for extent at index 0 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.655633327Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"mend","":"downstairs","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5302026-07-20T19:23:38.426Z{"msg":"extent 1 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3998, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3998, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3997, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.655660884Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"mend","":"downstairs","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5312026-07-20T19:23:38.426Z{"msg":"extent:1 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3998, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3998, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3997, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.655688382Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"mend","":"downstairs","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5322026-07-20T19:23:38.426Z{"msg":"found dest for source 0 for extent at index 1 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.655709281Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"mend","":"downstairs","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5332026-07-20T19:23:38.426Z{"msg":"extent 2 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3998, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3998, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3997, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.655736649Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"mend","":"downstairs","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5342026-07-20T19:23:38.427Z{"msg":"extent:2 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3998, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3998, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3997, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.655759537Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"mend","":"downstairs","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5352026-07-20T19:23:38.427Z{"msg":"found dest for source 0 for extent at index 2 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.655777116Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"mend","":"downstairs","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5362026-07-20T19:23:38.427Z{"msg":"extent 3 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3998, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3998, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3997, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.655798814Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"mend","":"downstairs","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5372026-07-20T19:23:38.427Z{"msg":"extent:3 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3998, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3998, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3997, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.655822272Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"mend","":"downstairs","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5382026-07-20T19:23:38.427Z{"msg":"found dest for source 0 for extent at index 3 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.655839431Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"mend","":"downstairs","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5392026-07-20T19:23:38.427Z{"msg":"extent 4 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3998, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3998, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3997, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.655862569Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"mend","":"downstairs","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5402026-07-20T19:23:38.427Z{"msg":"extent:4 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3998, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3998, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3997, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.655885007Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"mend","":"downstairs","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5412026-07-20T19:23:38.427Z{"msg":"found dest for source 0 for extent at index 4 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.655901806Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"mend","":"downstairs","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5422026-07-20T19:23:38.427Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.655931264Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"client":"0","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5432026-07-20T19:23:38.427Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.655948403Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"client":"1","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5442026-07-20T19:23:38.427Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.655964611Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"client":"2","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5452026-07-20T19:23:38.427Z{"msg":"Full repair list: {ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.656009658Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"downstairs","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5462026-07-20T19:23:38.427Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 3999, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 3999, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 3999, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 3999, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 3999, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.656156067Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"downstairs","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5472026-07-20T19:23:38.427Z{"msg":"starting reconciliation 654cb82b-45d8-4ad0-8033-4c42047428ee: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.656174875Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"downstairs","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5482026-07-20T19:23:38.427Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.656190584Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"client":"2","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5492026-07-20T19:23:38.432Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.665357744Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"client":"2","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5502026-07-20T19:23:38.435Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.667541698Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"client":"2","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5512026-07-20T19:23:38.435Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.669513787Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"client":"2","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5522026-07-20T19:23:38.438Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.67157744Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"client":"2","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5532026-07-20T19:23:38.438Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.673353814Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"downstairs","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5542026-07-20T19:23:38.438Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.673378772Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"client":"0","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5552026-07-20T19:23:38.438Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.673389041Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"client":"1","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5562026-07-20T19:23:38.438Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.67339903Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"client":"2","":"downstairs","session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5572026-07-20T19:23:38.438Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.673413769Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5582026-07-20T19:23:38.438Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.673423319Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5592026-07-20T19:23:38.438Z{"msg":"06490a9f-e383-470f-9ee1-a5ecdf9627fd is now active with session: e788d3c7-afab-4baf-a411-78857ade5e60","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.673437708Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"session_id":"e788d3c7-afab-4baf-a411-78857ade5e60"}
5602026-07-20T19:23:38.438Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.673446757Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195}
5612026-07-20T19:23:38.438Z{"msg":"Activated sub_volume 06490a9f-e383-470f-9ee1-a5ecdf9627fd","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.673496433Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195}
5622026-07-20T19:23:38.438Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.673565848Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195}
5632026-07-20T19:23:38.438Z{"msg":"06490a9f-e383-470f-9ee1-a5ecdf9627fd active request set","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.673578907Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5642026-07-20T19:23:38.438Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.673587926Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"client":"0","":"downstairs","session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5652026-07-20T19:23:38.438Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.673602325Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"client":"1","":"downstairs","session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5662026-07-20T19:23:38.438Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.673612044Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"client":"2","":"downstairs","session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5672026-07-20T19:23:38.438Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.673621993Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"io task","client":"2","":"downstairs","session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5682026-07-20T19:23:38.438Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.673632213Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"io task","client":"0","":"downstairs","session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5692026-07-20T19:23:38.438Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.673646712Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"io task","client":"1","":"downstairs","session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5702026-07-20T19:23:38.438Z{"msg":"ds_connection connected from Ok(127.0.0.1:38025)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.67366643Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"io task","client":"2","":"downstairs","session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5712026-07-20T19:23:38.438Z{"msg":"ds_connection connected from Ok(127.0.0.1:51624)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.673685599Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"io task","client":"1","":"downstairs","session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5722026-07-20T19:23:38.438Z{"msg":"ds_connection connected from Ok(127.0.0.1:48832)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.673696208Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"io task","client":"0","":"downstairs","session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5732026-07-20T19:23:38.442Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.674092798Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"client":"1","":"downstairs","session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5742026-07-20T19:23:38.442Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.674118996Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"client":"0","":"downstairs","session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5752026-07-20T19:23:38.442Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.674211808Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"client":"2","":"downstairs","session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5762026-07-20T19:23:38.442Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.674263305Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5772026-07-20T19:23:38.443Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.674293722Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5782026-07-20T19:23:38.443Z{"msg":"[0]R flush_numbers: [5317, 5317, 5317, 5317, 5317]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.674366037Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"downstairs","":"downstairs","session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5792026-07-20T19:23:38.443Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.674381745Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"downstairs","":"downstairs","session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5802026-07-20T19:23:38.443Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.674391855Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"downstairs","":"downstairs","session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5812026-07-20T19:23:38.443Z{"msg":"[1]R flush_numbers: [5317, 5317, 5317, 5317, 5317]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.674406014Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"downstairs","":"downstairs","session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5822026-07-20T19:23:38.443Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.674415573Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"downstairs","":"downstairs","session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5832026-07-20T19:23:38.443Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.674429712Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"downstairs","":"downstairs","session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5842026-07-20T19:23:38.443Z{"msg":"[2]R flush_numbers: [5317, 5317, 5317, 5317, 5317]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.674439431Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"downstairs","":"downstairs","session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5852026-07-20T19:23:38.443Z{"msg":"[2]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.67445343Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"downstairs","":"downstairs","session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5862026-07-20T19:23:38.443Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.674463669Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"downstairs","":"downstairs","session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5872026-07-20T19:23:38.443Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.674477408Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"downstairs","":"downstairs","session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5882026-07-20T19:23:38.443Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.674487257Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"downstairs","":"downstairs","session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5892026-07-20T19:23:38.443Z{"msg":"Next flush: 5318","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.674500816Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"downstairs","":"downstairs","session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5902026-07-20T19:23:38.443Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.674510316Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"":"downstairs","":"downstairs","session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5912026-07-20T19:23:38.443Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.674524345Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5922026-07-20T19:23:38.443Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.674533284Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"client":"0","":"downstairs","session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5932026-07-20T19:23:38.443Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.674547063Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"client":"1","":"downstairs","session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5942026-07-20T19:23:38.443Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.674556542Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"client":"2","":"downstairs","session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5952026-07-20T19:23:38.443Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.674570431Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5962026-07-20T19:23:38.443Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.67457988Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5972026-07-20T19:23:38.443Z{"msg":"06490a9f-e383-470f-9ee1-a5ecdf9627fd is now active with session: 148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.674594359Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5982026-07-20T19:23:38.443Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.674603009Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"session_id":"148f96a1-a4a0-41c2-b0c7-8b70b28f6ac5"}
5992026-07-20T19:23:38.443Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.674612168Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195}
6002026-07-20T19:23:38.443Z{"msg":"Activated sub_volume 06490a9f-e383-470f-9ee1-a5ecdf9627fd","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.674620397Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195}
6012026-07-20T19:23:38.443Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.674632976Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"task":"crutest"}
6022026-07-20T19:23:38.443ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
6032026-07-20T19:23:38.443ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
6042026-07-20T19:23:38.443ZRead and Verify all blocks (0..100 range:true)
6052026-07-20T19:23:38.487Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.722896502Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"task":"crutest"}
6062026-07-20T19:23:38.487Z{"msg":"Verify test completed","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.72292055Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"task":"crutest"}
6072026-07-20T19:23:38.491Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.722938389Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"task":"crutest"}
6082026-07-20T19:23:38.491Z{"msg":"CLIENT: Tests done. All submitted work has been ACK'd","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.722951578Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"task":"crutest"}
6092026-07-20T19:23:38.491Z----------------------------------------------------------------
6102026-07-20T19:23:38.491Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6112026-07-20T19:23:38.491ZStates: Active Active Active
6122026-07-20T19:23:38.491ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6132026-07-20T19:23:38.491Z 1000 Acked Read 50 Done Done Done false
6142026-07-20T19:23:38.491Z STATES DS:0 DS:1 DS:2 TOTAL
6152026-07-20T19:23:38.491Z Sent 0 0 0 0
6162026-07-20T19:23:38.491Z Done 1 1 1 3
6172026-07-20T19:23:38.491Z Skipped 0 0 0 0
6182026-07-20T19:23:38.491Z Error 0 0 0 0
6192026-07-20T19:23:38.491ZLast Flush: None None None
6202026-07-20T19:23:38.491ZDownstairs last five completed:
6212026-07-20T19:23:38.491ZUpstairs last five completed: 1000
6222026-07-20T19:23:38.491Z----------------------------------------------------------------
6232026-07-20T19:23:38.491Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6242026-07-20T19:23:38.491ZStates: Active Active Active
6252026-07-20T19:23:38.491ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6262026-07-20T19:23:38.491Z 1000 Acked Read 50 Done Done Done false
6272026-07-20T19:23:38.492Z STATES DS:0 DS:1 DS:2 TOTAL
6282026-07-20T19:23:38.492Z Sent 0 0 0 0
6292026-07-20T19:23:38.492Z Done 1 1 1 3
6302026-07-20T19:23:38.492Z Skipped 0 0 0 0
6312026-07-20T19:23:38.492Z Error 0 0 0 0
6322026-07-20T19:23:38.492ZLast Flush: None None None
6332026-07-20T19:23:38.492ZDownstairs last five completed:
6342026-07-20T19:23:38.492ZUpstairs last five completed: 1000
6352026-07-20T19:23:38.492Z{"msg":"CLIENT: Up:0 ds:2 act:6","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:36.723115376Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"task":"crutest"}
6362026-07-20T19:23:42.492Z----------------------------------------------------------------
6372026-07-20T19:23:42.492Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6382026-07-20T19:23:42.493ZDownstairs last five completed: 1001 1000
6392026-07-20T19:23:42.493ZUpstairs last five completed: 1001 1000
6402026-07-20T19:23:42.497Z----------------------------------------------------------------
6412026-07-20T19:23:42.497Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6422026-07-20T19:23:42.497ZDownstairs last five completed: 1001 1000
6432026-07-20T19:23:42.497ZUpstairs last five completed: 1001 1000
6442026-07-20T19:23:42.497Z{"msg":"CLIENT: Up:0 ds:0 act:6","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:40.724661701Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"task":"crutest"}
6452026-07-20T19:23:42.497Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:23:40.724702897Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195,"task":"crutest"}
6462026-07-20T19:23:42.497Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-20T19:23:40.724726546Z","hostname":"w-01KY0FE5VTQK6S1ZS8T3K93BSW","pid":1195}
6472026-07-20T19:23:42.497ZRepair part 2 passed
6482026-07-20T19:23:42.497Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
6492026-07-20T19:23:42.510ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6502026-07-20T19:23:42.510Z 0 000-009 32 32 32 3998 3998 3998 F F F
6512026-07-20T19:23:42.511Z 1 010-019 32 32 32 3998 3998 3998 F F F
6522026-07-20T19:23:42.511Z 2 020-029 32 32 32 3998 3998 3998 F F F
6532026-07-20T19:23:42.516Z 3 030-039 32 32 32 3998 3998 3998 F F F
6542026-07-20T19:23:42.516Z 4 040-049 32 32 32 3998 3998 3998 F F F
6552026-07-20T19:23:42.516ZMax gen: 32, Max flush: 3998
6562026-07-20T19:23:42.516Zdump test passed
6572026-07-20T19:23:42.516Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
6582026-07-20T19:23:42.525Z Extent 1
6592026-07-20T19:23:42.525ZGEN 32 32 32
6602026-07-20T19:23:42.525ZFLUSH_ID 3998 3998 3998
6612026-07-20T19:23:42.525ZDIRTY
6622026-07-20T19:23:42.525Z
6632026-07-20T19:23:42.531ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
6642026-07-20T19:23:42.531Z010 A A A A A A
6652026-07-20T19:23:42.531Z011 A A A A A A
6662026-07-20T19:23:42.531Z012 A A A A A A
6672026-07-20T19:23:42.531Z013 A A A A A A
6682026-07-20T19:23:42.531Z014 A A A A A A
6692026-07-20T19:23:42.531Z015 A A A A A A
6702026-07-20T19:23:42.531Z016 A A A A A A
6712026-07-20T19:23:42.531Z017 A A A A A A
6722026-07-20T19:23:42.531Z018 A A A A A A
6732026-07-20T19:23:42.531Z019 A A A A A A
6742026-07-20T19:23:42.536Zdump extent test passed
6752026-07-20T19:23:42.536Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
6762026-07-20T19:23:42.541ZExtent 2 Block in extent 0 Actual block 20
6772026-07-20T19:23:42.541Z
6782026-07-20T19:23:42.545Z DATA SHA256 VER
6792026-07-20T19:23:42.545Z------ ---------------------------------------------------------------- ---
6802026-07-20T19:23:42.545Z 0 b2ea67ca99f55fe9e67f0a105c1296137d006769d5a5b6751be983d63c262b07 A
6812026-07-20T19:23:42.545Z 1 b2ea67ca99f55fe9e67f0a105c1296137d006769d5a5b6751be983d63c262b07 A
6822026-07-20T19:23:42.545Z 2 b2ea67ca99f55fe9e67f0a105c1296137d006769d5a5b6751be983d63c262b07 A
6832026-07-20T19:23:42.545Z
6842026-07-20T19:23:42.545ZNONCES 0 1 2 DIFF
6852026-07-20T19:23:42.545Z------ ------------------------ ------------------------ ------------------------ -----
6862026-07-20T19:23:42.545Z 0 dc054803e8381c28472319f0 dc054803e8381c28472319f0 dc054803e8381c28472319f0
6872026-07-20T19:23:42.545Z
6882026-07-20T19:23:42.546Z TAGS 0 1 2 DIFF
6892026-07-20T19:23:42.546Z------ -------------------------------- -------------------------------- -------------------------------- -----
6902026-07-20T19:23:42.546Z 0 2cd1f5a1d0fb2d8fbc44f3f537af4bf2 2cd1f5a1d0fb2d8fbc44f3f537af4bf2 2cd1f5a1d0fb2d8fbc44f3f537af4bf2
6912026-07-20T19:23:42.546Z
6922026-07-20T19:23:42.546ZHASHES 0 1 2 DIFF
6932026-07-20T19:23:42.546Z------ ---------------- ---------------- ---------------- -----
6942026-07-20T19:23:42.546Z
6952026-07-20T19:23:42.546Zdump block test passed
6962026-07-20T19:23:42.546ZInitial upstairs tests have completed, stopping all downstairs
6972026-07-20T19:23:47.569ZCreating 7 larger downstairs regions
6982026-07-20T19:23:47.768ZStarting 7 downstairs
6992026-07-20T19:23:52.777Zdsc restarted at PID: 1211
7002026-07-20T19:23:52.872ZNow do the replace-reconcile test
7012026-07-20T19:23:52.886ZUsing 8870 for the replacement port
7022026-07-20T19:27:21.563ZCompleted test: replace-reconcile
7032026-07-20T19:27:21.573ZNow do the replace-before-active test
7042026-07-20T19:31:16.258ZCompleted test: replace-before-active
7052026-07-20T19:31:16.259ZAll tests have completed, stopping all downstairs
7062026-07-20T19:31:16.275Z
7072026-07-20T19:31:16.279ZAll Tests have passed
7082026-07-20T19:31:16.279Z11:34 Test duration
7092026-07-20T19:31:16.279Z
7102026-07-20T19:31:16.279Zreal 11:33.573220202
7112026-07-20T19:31:16.279Zuser 16:59.593580908
7122026-07-20T19:31:16.279Zsys 11.831858581
7132026-07-20T19:31:16.279Ztrap 0.426400822
7142026-07-20T19:31:16.280Ztflt 0.040878557
7152026-07-20T19:31:16.280Zdflt 0.014149723
7162026-07-20T19:31:16.280Zkflt 0.000965613
7172026-07-20T19:31:16.280Zlock 15:59:26.226483633
7182026-07-20T19:31:16.280Zslp 50:18.596351502
7192026-07-20T19:31:16.280Zlat 1:50.483289042
7202026-07-20T19:31:16.280Zstop 0.040253236
7212026-07-20T19:31:16.280Z+ echo 'test-up-2region-encrypted ends'
7222026-07-20T19:31:16.280Ztest-up-2region-encrypted ends
7232026-07-20T19:31:21.282Zprocess exited: duration 702286 ms, exit code 0
7242026-07-20T19:31:21.282Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
7252026-07-20T19:32:21.325Zexec warning: : stdout descriptor may be held open by a background process; giving up!
7262026-07-20T19:32:21.326Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
7272026-07-20T19:32:21.340Zfound 17 output files
7282026-07-20T19:32:21.340Zuploading: /tmp/test_up-build/dsc-out.txt (57952 bytes)
7292026-07-20T19:32:22.354Zuploaded: /tmp/test_up-build/dsc-out.txt
7302026-07-20T19:32:22.362Zuploading: /tmp/test_up-build/test_up_out.txt (23053583 bytes)
7312026-07-20T19:32:23.518Zuploaded: /tmp/test_up-build/test_up_out.txt
7322026-07-20T19:32:23.518Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (550975 bytes)
7332026-07-20T19:32:24.535Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
7342026-07-20T19:32:24.535Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (548320 bytes)
7352026-07-20T19:32:25.552Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
7362026-07-20T19:32:25.553Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (978612 bytes)
7372026-07-20T19:32:26.571Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
7382026-07-20T19:32:26.571Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (543036 bytes)
7392026-07-20T19:32:27.590Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
7402026-07-20T19:32:27.590Zuploading: /tmp/test_up-build/dsc/downstairs-8850.txt (1343665 bytes)
7412026-07-20T19:32:28.611Zuploaded: /tmp/test_up-build/dsc/downstairs-8850.txt
7422026-07-20T19:32:28.611Zuploading: /tmp/test_up-build/dsc/downstairs-8860.txt (47322 bytes)
7432026-07-20T19:32:29.624Zuploaded: /tmp/test_up-build/dsc/downstairs-8860.txt
7442026-07-20T19:32:29.624Zuploading: /tmp/test_up-build/dsc/downstairs-8870.txt (3584172 bytes)
7452026-07-20T19:32:30.661Zuploaded: /tmp/test_up-build/dsc/downstairs-8870.txt
7462026-07-20T19:32:30.662Zuploading: /tmp/debug/df.txt (1270 bytes)
7472026-07-20T19:32:31.670Zuploaded: /tmp/debug/df.txt
7482026-07-20T19:32:31.674Zuploading: /tmp/debug/dtrace.txt (1033334 bytes)
7492026-07-20T19:32:32.688Zuploaded: /tmp/debug/dtrace.txt
7502026-07-20T19:32:32.688Zuploading: /tmp/debug/iostat.txt (225684 bytes)
7512026-07-20T19:32:32.697Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 225684 -> 228951
7522026-07-20T19:32:33.705Zuploaded: /tmp/debug/iostat.txt
7532026-07-20T19:32:33.705Zuploading: /tmp/debug/mpstat.txt (551216 bytes)
7542026-07-20T19:32:33.717Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 551216 -> 560654
7552026-07-20T19:32:34.724Zuploaded: /tmp/debug/mpstat.txt
7562026-07-20T19:32:34.724Zuploading: /tmp/debug/paging.txt (97016 bytes)
7572026-07-20T19:32:34.731Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 97016 -> 98900
7582026-07-20T19:32:35.737Zuploaded: /tmp/debug/paging.txt
7592026-07-20T19:32:35.737Zuploading: /tmp/debug/prstat.txt (1057588 bytes)
7602026-07-20T19:32:35.753Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 1057588 -> 1075703
7612026-07-20T19:32:36.759Zuploaded: /tmp/debug/prstat.txt
7622026-07-20T19:32:36.759Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
7632026-07-20T19:32:37.767Zuploaded: /tmp/debug/psrinfo.txt
7642026-07-20T19:32:37.767Zuploading: /tmp/debug/upstairs-info.txt (163625 bytes)
7652026-07-20T19:32:38.783Zuploaded: /tmp/debug/upstairs-info.txt