01KN2HMMVBY8BC2SVV8X03ZNX0: test-up-2region-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01KN2HN1DNNW5NE4XCJTHG3ESW

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12026-03-31T18:30:23.834Zjob dependencies complete; ready to run (waiting for 16 m 4 s)
22026-03-31T18:32:54.383Zjob assigned to worker 01KN2JNFQKF115ANF9FAMMJH5X [factory aws, i-07a5c3fd687ba5f35] (queued for 2 m 30 s)
32026-03-31T18:33:01.061Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42026-03-31T18:33:04.068Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52026-03-31T18:33:04.072Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62026-03-31T18:33:05.523Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72026-03-31T18:33:05.526Zdownloading input: /input/build/work/bins/crutest.gz
82026-03-31T18:33:07.230Zdownloaded input: /input/build/work/bins/crutest.gz
92026-03-31T18:33:07.236Zdownloading input: /input/build/work/bins/dsc.gz
102026-03-31T18:33:08.114Zdownloaded input: /input/build/work/bins/dsc.gz
112026-03-31T18:33:08.117Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122026-03-31T18:33:08.146Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132026-03-31T18:33:08.150Zdownloading input: /input/build/work/scripts/test_ds.sh
142026-03-31T18:33:08.179Zdownloaded input: /input/build/work/scripts/test_ds.sh
152026-03-31T18:33:08.183Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162026-03-31T18:33:08.208Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172026-03-31T18:33:08.208Zdownloading input: /input/build/work/scripts/test_repair.sh
182026-03-31T18:33:08.238Zdownloaded input: /input/build/work/scripts/test_repair.sh
192026-03-31T18:33:08.238Zdownloading input: /input/build/work/scripts/test_replay.sh
202026-03-31T18:33:08.264Zdownloaded input: /input/build/work/scripts/test_replay.sh
212026-03-31T18:33:08.264Zdownloading input: /input/build/work/scripts/test_up.sh
222026-03-31T18:33:08.295Zdownloaded input: /input/build/work/scripts/test_up.sh
232026-03-31T18:33:08.295Zdownloading input: /input/build/work/scripts/upstairs_info.d
242026-03-31T18:33:08.320Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252026-03-31T18:33:08.323Zdownloading input: /input/build/tmp/cargo-test-out.log
262026-03-31T18:33:08.920Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272026-03-31T18:33:08.920Zstarting task 0: "setup"
282026-03-31T18:33:08.926Z++ uname -s
292026-03-31T18:33:08.930Z+ kern=SunOS
302026-03-31T18:33:08.931Z+ build_user=build
312026-03-31T18:33:08.931Z+ build_uid=12345
322026-03-31T18:33:08.931Z+ work_dir=/work
332026-03-31T18:33:08.931Z+ input_dir=/input
342026-03-31T18:33:08.931Z+ [[ 0 == 12345 ]]
352026-03-31T18:33:08.931Z+ case "$kern" in
362026-03-31T18:33:08.931Z+ groupadd -g 12345 build
372026-03-31T18:33:08.931Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382026-03-31T18:33:10.939Z+ zfs create -o mountpoint=/work rpool/work
392026-03-31T18:33:11.236Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402026-03-31T18:33:11.241Z+ home_fs=zfs
412026-03-31T18:33:11.241Z+ [[ zfs == autofs ]]
422026-03-31T18:33:11.241Z+ mkdir -p /home/build
432026-03-31T18:33:11.241Z+ chown build:build /home/build /work
442026-03-31T18:33:13.239Z+ chmod 0700 /home/build /work
452026-03-31T18:33:13.242Zprocess exited: duration 4320 ms, exit code 0
 
462026-03-31T18:33:13.249Zstarting task 1: "authentication"
472026-03-31T18:33:13.267Zprocess exited: duration 18 ms, exit code 0
 
482026-03-31T18:33:13.275Zstarting task 2: "build"
492026-03-31T18:33:13.280Z+ banner cores
502026-03-31T18:33:13.283Z
512026-03-31T18:33:13.283Z #### #### ##### ###### ####
522026-03-31T18:33:13.283Z # # # # # # # #
532026-03-31T18:33:13.283Z # # # # # ##### ####
542026-03-31T18:33:13.283Z # # # ##### # #
552026-03-31T18:33:13.283Z # # # # # # # # #
562026-03-31T18:33:13.283Z #### #### # # ###### ####
572026-03-31T18:33:13.283Z
582026-03-31T18:33:13.283Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
592026-03-31T18:33:13.286Z+ banner unpack
602026-03-31T18:33:13.286Z
612026-03-31T18:33:13.287Z # # # # ##### ## #### # #
622026-03-31T18:33:13.287Z # # ## # # # # # # # # #
632026-03-31T18:33:13.287Z # # # # # # # # # # ####
642026-03-31T18:33:13.287Z # # # # # ##### ###### # # #
652026-03-31T18:33:13.287Z # # # ## # # # # # # #
662026-03-31T18:33:13.287Z #### # # # # # #### # #
672026-03-31T18:33:13.287Z
682026-03-31T18:33:13.287Z+ mkdir -p /var/tmp/bins
692026-03-31T18:33:13.290Z+ for t in "$input/bins/"*.gz
702026-03-31T18:33:13.290Z++ basename /input/build/work/bins/crucible-downstairs.gz
712026-03-31T18:33:13.290Z+ b=crucible-downstairs.gz
722026-03-31T18:33:13.290Z+ b=crucible-downstairs
732026-03-31T18:33:13.290Z+ gunzip
742026-03-31T18:33:15.865Z+ chmod +x /var/tmp/bins/crucible-downstairs
752026-03-31T18:33:15.869Z+ for t in "$input/bins/"*.gz
762026-03-31T18:33:15.869Z++ basename /input/build/work/bins/crucible-hammer.gz
772026-03-31T18:33:15.869Z+ b=crucible-hammer.gz
782026-03-31T18:33:15.869Z+ b=crucible-hammer
792026-03-31T18:33:15.869Z+ gunzip
802026-03-31T18:33:18.207Z+ chmod +x /var/tmp/bins/crucible-hammer
812026-03-31T18:33:18.213Z+ for t in "$input/bins/"*.gz
822026-03-31T18:33:18.213Z++ basename /input/build/work/bins/crutest.gz
832026-03-31T18:33:18.213Z+ b=crutest.gz
842026-03-31T18:33:18.213Z+ b=crutest
852026-03-31T18:33:18.213Z+ gunzip
862026-03-31T18:33:20.652Z+ chmod +x /var/tmp/bins/crutest
872026-03-31T18:33:20.658Z+ for t in "$input/bins/"*.gz
882026-03-31T18:33:20.658Z++ basename /input/build/work/bins/dsc.gz
892026-03-31T18:33:20.658Z+ b=dsc.gz
902026-03-31T18:33:20.658Z+ b=dsc
912026-03-31T18:33:20.658Z+ gunzip
922026-03-31T18:33:21.825Z+ chmod +x /var/tmp/bins/dsc
932026-03-31T18:33:21.828Z+ export BINDIR=/var/tmp/bins
942026-03-31T18:33:21.828Z+ BINDIR=/var/tmp/bins
952026-03-31T18:33:21.829Z+ jobpid=1088
962026-03-31T18:33:21.829Z+ echo 'Setup debug logging'
972026-03-31T18:33:21.829ZSetup debug logging
982026-03-31T18:33:21.829Z+ mkdir /tmp/debug
992026-03-31T18:33:21.829Z+ sleep 3600
1002026-03-31T18:33:21.829Z+ psrinfo -v
1012026-03-31T18:33:21.832Z+ df -h
1022026-03-31T18:33:21.836Z+ prstat -d d -mLc 1
1032026-03-31T18:33:21.839Z+ iostat -T d -xn 1
1042026-03-31T18:33:21.839Z+ mpstat -T d 1
1052026-03-31T18:33:21.839Z+ vmstat -T d -p 1
1062026-03-31T18:33:21.839Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1072026-03-31T18:33:21.839Z+ banner test_up_2r_encrypted
1082026-03-31T18:33:21.839Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1092026-03-31T18:33:21.843Z #####
1102026-03-31T18:33:21.843Z ##### ###### #### ##### # # ##### # # #####
1112026-03-31T18:33:21.843Z # # # # # # # # # # #
1122026-03-31T18:33:21.843Z # ##### #### # # # # # ##### # #
1132026-03-31T18:33:21.843Z # # # # # # ##### # #####
1142026-03-31T18:33:21.843Z # # # # # # # # # # #
1152026-03-31T18:33:21.843Z # ###### #### # ####### #### # ####### ####### # #
1162026-03-31T18:33:21.843Z
1172026-03-31T18:33:21.843Z+ ptime -m bash /input/build/work/scripts/test_up.sh -r 2 -N encrypted
1182026-03-31T18:33:21.848Z/input/build/work
1192026-03-31T18:33:21.856ZUsing 2 region sets
1202026-03-31T18:33:21.856ZTurn off color for downstairs dump
1212026-03-31T18:33:21.868ZUpstairs using key: tE5JraOhcLCGMrtLvW4ovkqCD3iISXGWjE77AlqaQjk=
1222026-03-31T18:33:21.871Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1232026-03-31T18:33:21.871ZCreating 6 downstairs regions
1242026-03-31T18:33:22.113ZStarting 6 downstairs
1252026-03-31T18:33:22.118Zdsc started at PID: 1136
1262026-03-31T18:33:27.205ZDisable automatic restart on all downstairs
1272026-03-31T18:33:27.219Z
1282026-03-31T18:33:27.219ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1292026-03-31T18:33:27.222ZRunning test: span
1302026-03-31T18:33:31.324ZCompleted test: span
1312026-03-31T18:33:36.331ZRunning test: big
1322026-03-31T18:33:40.638ZCompleted test: big
1332026-03-31T18:33:45.644ZRunning test: dep
1342026-03-31T18:34:05.334ZCompleted test: dep
1352026-03-31T18:34:10.341ZRunning test: balloon
1362026-03-31T18:36:37.667ZCompleted test: balloon
1372026-03-31T18:36:42.675ZRunning test: deactivate
1382026-03-31T18:36:47.007ZCompleted test: deactivate
1392026-03-31T18:36:52.014ZRunning hammer
1402026-03-31T18:36:59.364ZRun repair tests
1412026-03-31T18:36:59.368Z/var/tmp/bins/crutest fill -g 31 -q --key tE5JraOhcLCGMrtLvW4ovkqCD3iISXGWjE77AlqaQjk= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1422026-03-31T18:36:59.386Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.89223781Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"task":"crutest"}
1432026-03-31T18:36:59.391Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.892451401Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"task":"crutest"}
1442026-03-31T18:36:59.391Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.893035602Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"task":"crutest"}
1452026-03-31T18:36:59.391Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.893303036Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179}
1462026-03-31T18:36:59.391Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"15d1a4bfe3549686eb976ff56710f1466c0e0e4a\",\n git_commit_timestamp: \"2026-03-31T18:13:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.893339361Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179}
1472026-03-31T18:36:59.391Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.89334925Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179}
1482026-03-31T18:36:59.391Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.893361998Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179}
1492026-03-31T18:36:59.391Z{"msg":"Crucible 7fb5af44-7c0b-4f61-9d49-b890f4b4cf98 has session id: ace4469b-7732-4a2f-85f2-d17c722f7b6c","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.893678875Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1502026-03-31T18:36:59.392Z{"msg":"Upstairs opts: Upstairs UUID: 7fb5af44-7c0b-4f61-9d49-b890f4b4cf98, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.893705872Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1512026-03-31T18:36:59.392Z{"msg":"Crucible stats registered with UUID: 7fb5af44-7c0b-4f61-9d49-b890f4b4cf98","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.89371683Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1522026-03-31T18:36:59.392Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-31T18:36:57.893726979Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1532026-03-31T18:36:59.392Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.89386611Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"io task","client":"0","":"downstairs","session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1542026-03-31T18:36:59.392Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.893904065Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"io task","client":"2","":"downstairs","session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1552026-03-31T18:36:59.392Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.893915723Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"io task","client":"1","":"downstairs","session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1562026-03-31T18:36:59.392Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.894666762Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"task":"crutest"}
1572026-03-31T18:36:59.392Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.894737332Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179}
1582026-03-31T18:36:59.392Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"15d1a4bfe3549686eb976ff56710f1466c0e0e4a\",\n git_commit_timestamp: \"2026-03-31T18:13:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.894758869Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179}
1592026-03-31T18:36:59.392Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.894768128Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179}
1602026-03-31T18:36:59.392Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.894780596Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179}
1612026-03-31T18:36:59.392Z{"msg":"Crucible 7fb5af44-7c0b-4f61-9d49-b890f4b4cf98 has session id: 362b6730-abb1-4d10-a336-fca88d110a51","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.894961512Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
1622026-03-31T18:36:59.392Z{"msg":"Upstairs opts: Upstairs UUID: 7fb5af44-7c0b-4f61-9d49-b890f4b4cf98, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.894996507Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
1632026-03-31T18:36:59.392Z{"msg":"Crucible stats registered with UUID: 7fb5af44-7c0b-4f61-9d49-b890f4b4cf98","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.895008425Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
1642026-03-31T18:36:59.392Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-31T18:36:57.895018984Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
1652026-03-31T18:36:59.392Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.895028803Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"io task","client":"0","":"downstairs","session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
1662026-03-31T18:36:59.392Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.895039861Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"io task","client":"2","":"downstairs","session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
1672026-03-31T18:36:59.392Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.8950505Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"io task","client":"1","":"downstairs","session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
1682026-03-31T18:36:59.392Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.895149896Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"task":"crutest"}
1692026-03-31T18:36:59.392Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.895165854Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179}
1702026-03-31T18:36:59.392Z{"msg":"7fb5af44-7c0b-4f61-9d49-b890f4b4cf98 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.895177063Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1712026-03-31T18:36:59.392Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.895186461Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"client":"0","":"downstairs","session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1722026-03-31T18:36:59.392Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.895200829Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"client":"1","":"downstairs","session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1732026-03-31T18:36:59.392Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.895211148Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"client":"2","":"downstairs","session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1742026-03-31T18:36:59.392Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.895221317Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"io task","client":"0","":"downstairs","session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1752026-03-31T18:36:59.392Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.895231805Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"io task","client":"2","":"downstairs","session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1762026-03-31T18:36:59.393Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.895242334Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"io task","client":"1","":"downstairs","session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1772026-03-31T18:36:59.393Z{"msg":"ds_connection connected from Ok(127.0.0.1:59726)","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.895315654Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"io task","client":"0","":"downstairs","session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1782026-03-31T18:36:59.393Z{"msg":"ds_connection connected from Ok(127.0.0.1:44421)","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.895326922Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"io task","client":"2","":"downstairs","session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1792026-03-31T18:36:59.393Z{"msg":"ds_connection connected from Ok(127.0.0.1:58275)","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.895337491Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"io task","client":"1","":"downstairs","session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1802026-03-31T18:36:59.393Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.895912633Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"client":"2","":"downstairs","session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1812026-03-31T18:36:59.398Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.895964606Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"client":"1","":"downstairs","session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1822026-03-31T18:36:59.398Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896014429Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"client":"0","":"downstairs","session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1832026-03-31T18:36:59.398Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896140322Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1842026-03-31T18:36:59.398Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896150301Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1852026-03-31T18:36:59.398Z{"msg":"[0]R flush_numbers: [4003, 4003, 4003, 4003, 4003]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896184646Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"downstairs","":"downstairs","session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1862026-03-31T18:36:59.398Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896195695Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"downstairs","":"downstairs","session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1872026-03-31T18:36:59.398Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896206133Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"downstairs","":"downstairs","session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1882026-03-31T18:36:59.398Z{"msg":"[1]R flush_numbers: [4003, 4003, 4003, 4003, 4003]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896217082Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"downstairs","":"downstairs","session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1892026-03-31T18:36:59.398Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.89622761Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"downstairs","":"downstairs","session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1902026-03-31T18:36:59.398Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896238359Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"downstairs","":"downstairs","session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1912026-03-31T18:36:59.398Z{"msg":"[2]R flush_numbers: [4003, 4003, 4003, 4003, 4003]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896249058Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"downstairs","":"downstairs","session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1922026-03-31T18:36:59.398Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896259586Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"downstairs","":"downstairs","session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1932026-03-31T18:36:59.398Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896270265Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"downstairs","":"downstairs","session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1942026-03-31T18:36:59.398Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896280283Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"downstairs","":"downstairs","session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1952026-03-31T18:36:59.398Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896290652Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"downstairs","":"downstairs","session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1962026-03-31T18:36:59.399Z{"msg":"Next flush: 4004","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896300521Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"downstairs","":"downstairs","session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1972026-03-31T18:36:59.399Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896310529Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"downstairs","":"downstairs","session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1982026-03-31T18:36:59.399Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896320858Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
1992026-03-31T18:36:59.399Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896330167Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"client":"0","":"downstairs","session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
2002026-03-31T18:36:59.399Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896340285Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"client":"1","":"downstairs","session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
2012026-03-31T18:36:59.399Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896350244Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"client":"2","":"downstairs","session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
2022026-03-31T18:36:59.399Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896360602Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
2032026-03-31T18:36:59.399Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896370331Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
2042026-03-31T18:36:59.399Z{"msg":"7fb5af44-7c0b-4f61-9d49-b890f4b4cf98 is now active with session: ace4469b-7732-4a2f-85f2-d17c722f7b6c","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.89638035Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
2052026-03-31T18:36:59.399Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896389498Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"session_id":"ace4469b-7732-4a2f-85f2-d17c722f7b6c"}
2062026-03-31T18:36:59.399Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896398857Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179}
2072026-03-31T18:36:59.399Z{"msg":"Activated sub_volume 7fb5af44-7c0b-4f61-9d49-b890f4b4cf98","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896411675Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179}
2082026-03-31T18:36:59.399Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896420094Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179}
2092026-03-31T18:36:59.399Z{"msg":"7fb5af44-7c0b-4f61-9d49-b890f4b4cf98 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896428703Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2102026-03-31T18:36:59.399Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.89652791Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"client":"0","":"downstairs","session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2112026-03-31T18:36:59.399Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896551597Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"client":"1","":"downstairs","session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2122026-03-31T18:36:59.399Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896568714Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"client":"2","":"downstairs","session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2132026-03-31T18:36:59.399Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896579253Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"io task","client":"2","":"downstairs","session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2142026-03-31T18:36:59.399Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896590011Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"io task","client":"0","":"downstairs","session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2152026-03-31T18:36:59.399Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.89660095Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"io task","client":"1","":"downstairs","session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2162026-03-31T18:36:59.399Z{"msg":"ds_connection connected from Ok(127.0.0.1:35264)","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896611928Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"io task","client":"2","":"downstairs","session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2172026-03-31T18:36:59.399Z{"msg":"ds_connection connected from Ok(127.0.0.1:46931)","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896629126Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"io task","client":"0","":"downstairs","session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2182026-03-31T18:36:59.400Z{"msg":"ds_connection connected from Ok(127.0.0.1:44633)","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896639975Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"io task","client":"1","":"downstairs","session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2192026-03-31T18:36:59.400Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896799003Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"client":"0","":"downstairs","session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2202026-03-31T18:36:59.400Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.8968235Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"client":"1","":"downstairs","session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2212026-03-31T18:36:59.400Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896869394Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"client":"2","":"downstairs","session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2222026-03-31T18:36:59.400Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896959931Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2232026-03-31T18:36:59.400Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896985268Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2242026-03-31T18:36:59.400Z{"msg":"[0]R flush_numbers: [5307, 5321, 5321, 5315, 5319]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.896994777Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"downstairs","":"downstairs","session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2252026-03-31T18:36:59.400Z{"msg":"[0]R generation: [10, 14, 14, 13, 13]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.897005075Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"downstairs","":"downstairs","session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2262026-03-31T18:36:59.400Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.897020713Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"downstairs","":"downstairs","session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2272026-03-31T18:36:59.400Z{"msg":"[1]R flush_numbers: [5307, 5321, 5321, 5315, 5319]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.897031472Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"downstairs","":"downstairs","session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2282026-03-31T18:36:59.400Z{"msg":"[1]R generation: [10, 14, 14, 13, 13]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.89704546Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"downstairs","":"downstairs","session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2292026-03-31T18:36:59.400Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.897055688Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"downstairs","":"downstairs","session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2302026-03-31T18:36:59.400Z{"msg":"[2]R flush_numbers: [5307, 5321, 5321, 5315, 5319]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.897066397Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"downstairs","":"downstairs","session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2312026-03-31T18:36:59.400Z{"msg":"[2]R generation: [10, 14, 14, 13, 13]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.897076576Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"downstairs","":"downstairs","session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2322026-03-31T18:36:59.400Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.897087264Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"downstairs","":"downstairs","session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2332026-03-31T18:36:59.400Z{"msg":"Max found gen is 15","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.897097483Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"downstairs","":"downstairs","session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2342026-03-31T18:36:59.400Z{"msg":"Generation requested: 31 >= found:15","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.897107771Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
2352026-03-31T18:36:59.401Z362b6730-abb1-4d10-a336-fca88d110a51"}
2362026-03-31T18:36:59.401Z{"msg":"Next flush: 5322","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.897122119Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"downstairs","":"downstairs","session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2372026-03-31T18:36:59.401Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.897132148Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"":"downstairs","":"downstairs","session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2382026-03-31T18:36:59.401Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.897142347Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2392026-03-31T18:36:59.401Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.897151735Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"client":"0","":"downstairs","session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2402026-03-31T18:36:59.401Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.897162084Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"client":"1","":"downstairs","session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2412026-03-31T18:36:59.401Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.897172393Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"client":"2","":"downstairs","session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2422026-03-31T18:36:59.401Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.897182831Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2432026-03-31T18:36:59.401Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.89719276Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2442026-03-31T18:36:59.401Z{"msg":"7fb5af44-7c0b-4f61-9d49-b890f4b4cf98 is now active with session: 362b6730-abb1-4d10-a336-fca88d110a51","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.897203198Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2452026-03-31T18:36:59.401Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.897212427Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179,"session_id":"362b6730-abb1-4d10-a336-fca88d110a51"}
2462026-03-31T18:36:59.401Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.897222336Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179}
2472026-03-31T18:36:59.401Z{"msg":"Activated sub_volume 7fb5af44-7c0b-4f61-9d49-b890f4b4cf98","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:36:57.897231265Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179}
2482026-03-31T18:36:59.401ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
2492026-03-31T18:36:59.401ZFill test
2502026-03-31T18:36:59.417ZRead and Verify all blocks (0..100 range:false)
2512026-03-31T18:36:59.464ZWrote out file "/var/tmp/test_up-build/verify_file"
2522026-03-31T18:36:59.464ZCLIENT: Tests done. All submitted work has been ACK'd
2532026-03-31T18:36:59.468Z----------------------------------------------------------------
2542026-03-31T18:36:59.468Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2552026-03-31T18:36:59.468ZStates: Active Active Active
2562026-03-31T18:36:59.468ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2572026-03-31T18:36:59.468Z 1002 Acked Read 50 Done Done Done false
2582026-03-31T18:36:59.468Z STATES DS:0 DS:1 DS:2 TOTAL
2592026-03-31T18:36:59.468Z Sent 0 0 0 0
2602026-03-31T18:36:59.468Z Done 1 1 1 3
2612026-03-31T18:36:59.468Z Skipped 0 0 0 0
2622026-03-31T18:36:59.468Z Error 0 0 0 0
2632026-03-31T18:36:59.468ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2642026-03-31T18:36:59.468ZDownstairs last five completed: 1001 1000
2652026-03-31T18:36:59.468ZUpstairs last five completed: 1002 1001 1000
2662026-03-31T18:36:59.468Z----------------------------------------------------------------
2672026-03-31T18:36:59.468Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2682026-03-31T18:36:59.468ZStates: Active Active Active
2692026-03-31T18:36:59.468ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2702026-03-31T18:36:59.468Z 1002 Acked Read 50 Done Done Done false
2712026-03-31T18:36:59.468Z STATES DS:0 DS:1 DS:2 TOTAL
2722026-03-31T18:36:59.468Z Sent 0 0 0 0
2732026-03-31T18:36:59.468Z Done 1 1 1 3
2742026-03-31T18:36:59.468Z Skipped 0 0 0 0
2752026-03-31T18:36:59.468Z Error 0 0 0 0
2762026-03-31T18:36:59.468ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2772026-03-31T18:36:59.469ZDownstairs last five completed: 1001 1000
2782026-03-31T18:36:59.469ZUpstairs last five completed: 1002 1001 1000
2792026-03-31T18:36:59.469ZCLIENT: Up:0 ds:2 act:6
2802026-03-31T18:37:03.466Z----------------------------------------------------------------
2812026-03-31T18:37:03.466Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2822026-03-31T18:37:03.466ZDownstairs last five completed: 1003 1002 1001 1000
2832026-03-31T18:37:03.466ZUpstairs last five completed: 1003 1002 1001 1000
2842026-03-31T18:37:03.469Z----------------------------------------------------------------
2852026-03-31T18:37:03.469Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2862026-03-31T18:37:03.469ZDownstairs last five completed: 1003 1002 1001 1000
2872026-03-31T18:37:03.469ZUpstairs last five completed: 1003 1002 1001 1000
2882026-03-31T18:37:03.469ZCLIENT: Up:0 ds:0 act:6
2892026-03-31T18:37:03.469ZCLIENT: All crucible jobs finished, exiting program
2902026-03-31T18:37:03.469Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-31T18:37:01.971747011Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1179}
2912026-03-31T18:37:03.475ZRepair setup passed
2922026-03-31T18:37:03.480ZCopy the region for /var/tmp/test_up-build/8830
2932026-03-31T18:37:03.480Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2942026-03-31T18:37:03.483Z/var/tmp/bins/crutest fill -g 32 -q --key tE5JraOhcLCGMrtLvW4ovkqCD3iISXGWjE77AlqaQjk= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2952026-03-31T18:37:03.498Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.003698777Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"task":"crutest"}
2962026-03-31T18:37:03.503Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.00396854Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"task":"crutest"}
2972026-03-31T18:37:03.503Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.004654077Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"task":"crutest"}
2982026-03-31T18:37:03.503Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.0049309Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185}
2992026-03-31T18:37:03.503Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"15d1a4bfe3549686eb976ff56710f1466c0e0e4a\",\n git_commit_timestamp: \"2026-03-31T18:13:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.004959326Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185}
3002026-03-31T18:37:03.503Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.004972764Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185}
3012026-03-31T18:37:03.503Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.004985162Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185}
3022026-03-31T18:37:03.503Z{"msg":"Crucible 44516ed0-5555-41a8-a055-23b4a503eb2c has session id: f17e1ace-7488-4093-bac2-78b7ec985453","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.005473066Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3032026-03-31T18:37:03.503Z{"msg":"Upstairs opts: Upstairs UUID: 44516ed0-5555-41a8-a055-23b4a503eb2c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.005499483Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3042026-03-31T18:37:03.503Z{"msg":"Crucible stats registered with UUID: 44516ed0-5555-41a8-a055-23b4a503eb2c","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.005509891Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3052026-03-31T18:37:03.503Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-31T18:37:02.00551983Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3062026-03-31T18:37:03.504Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.005658811Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3072026-03-31T18:37:03.504Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.005683848Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"io task","client":"1","":"downstairs","session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3082026-03-31T18:37:03.504Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.005694217Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3092026-03-31T18:37:03.504Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.006578517Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"task":"crutest"}
3102026-03-31T18:37:03.504Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.006636859Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185}
3112026-03-31T18:37:03.504Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"15d1a4bfe3549686eb976ff56710f1466c0e0e4a\",\n git_commit_timestamp: \"2026-03-31T18:13:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.006659816Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185}
3122026-03-31T18:37:03.504Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.006672314Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185}
3132026-03-31T18:37:03.504Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.006683923Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185}
3142026-03-31T18:37:03.504Z{"msg":"Crucible 44516ed0-5555-41a8-a055-23b4a503eb2c has session id: ca701710-b404-4f91-8128-fae25eaa6497","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.006895654Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3152026-03-31T18:37:03.504Z{"msg":"Upstairs opts: Upstairs UUID: 44516ed0-5555-41a8-a055-23b4a503eb2c, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.00692346Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3162026-03-31T18:37:03.504Z{"msg":"Crucible stats registered with UUID: 44516ed0-5555-41a8-a055-23b4a503eb2c","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.006942588Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3172026-03-31T18:37:03.504Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-31T18:37:02.006952336Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3182026-03-31T18:37:03.504Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.006965305Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3192026-03-31T18:37:03.504Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.006975813Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"io task","client":"1","":"downstairs","session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3202026-03-31T18:37:03.504Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.006986152Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3212026-03-31T18:37:03.504Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.007119234Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"task":"crutest"}
3222026-03-31T18:37:03.504Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.007135881Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185}
3232026-03-31T18:37:03.504Z{"msg":"44516ed0-5555-41a8-a055-23b4a503eb2c active request set","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.007159838Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3242026-03-31T18:37:03.504Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.007169827Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"client":"0","":"downstairs","session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3252026-03-31T18:37:03.504Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.007187404Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"client":"1","":"downstairs","session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3262026-03-31T18:37:03.504Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.007201782Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"client":"2","":"downstairs","session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3272026-03-31T18:37:03.504Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.007211671Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3282026-03-31T18:37:03.504Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.00722253Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3292026-03-31T18:37:03.504Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.007240287Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"io task","client":"1","":"downstairs","session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3302026-03-31T18:37:03.504Z{"msg":"ds_connection connected from Ok(127.0.0.1:41457)","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.007321696Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"io task","client":"1","":"downstairs","session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3312026-03-31T18:37:03.504Z{"msg":"ds_connection connected from Ok(127.0.0.1:35633)","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.007358991Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3322026-03-31T18:37:03.504Z{"msg":"ds_connection connected from Ok(127.0.0.1:44310)","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.007411674Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3332026-03-31T18:37:03.509Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008055977Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"client":"0","":"downstairs","session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3342026-03-31T18:37:03.509Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008091532Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"client":"1","":"downstairs","session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3352026-03-31T18:37:03.509Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008145595Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"client":"2","":"downstairs","session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3362026-03-31T18:37:03.509Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008281786Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3372026-03-31T18:37:03.509Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008339878Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3382026-03-31T18:37:03.509Z{"msg":"[0]R flush_numbers: [4004, 4004, 4004, 4004, 4004]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008378883Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"downstairs","":"downstairs","session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3392026-03-31T18:37:03.509Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008394141Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"downstairs","":"downstairs","session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3402026-03-31T18:37:03.509Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.00840415Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"downstairs","":"downstairs","session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3412026-03-31T18:37:03.509Z{"msg":"[1]R flush_numbers: [4004, 4004, 4004, 4004, 4004]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008418858Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"downstairs","":"downstairs","session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3422026-03-31T18:37:03.509Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008428866Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"downstairs","":"downstairs","session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3432026-03-31T18:37:03.509Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008450374Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"downstairs","":"downstairs","session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3442026-03-31T18:37:03.509Z{"msg":"[2]R flush_numbers: [4004, 4004, 4004, 4004, 4004]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008460912Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"downstairs","":"downstairs","session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3452026-03-31T18:37:03.509Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008471221Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"downstairs","":"downstairs","session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3462026-03-31T18:37:03.509Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008485719Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"downstairs","":"downstairs","session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3472026-03-31T18:37:03.509Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008495417Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"downstairs","":"downstairs","session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3482026-03-31T18:37:03.509Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008505546Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"downstairs","":"downstairs","session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3492026-03-31T18:37:03.509Z{"msg":"Next flush: 4005","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008519084Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"downstairs","":"downstairs","session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3502026-03-31T18:37:03.509Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008528723Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"downstairs","":"downstairs","session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3512026-03-31T18:37:03.509Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008538422Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3522026-03-31T18:37:03.509Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.0085514Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"client":"0","":"downstairs","session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3532026-03-31T18:37:03.509Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008561399Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"client":"1","":"downstairs","session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3542026-03-31T18:37:03.509Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008570947Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"client":"2","":"downstairs","session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3552026-03-31T18:37:03.509Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008580786Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3562026-03-31T18:37:03.509Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008590145Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3572026-03-31T18:37:03.509Z{"msg":"44516ed0-5555-41a8-a055-23b4a503eb2c is now active with session: f17e1ace-7488-4093-bac2-78b7ec985453","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008604333Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3582026-03-31T18:37:03.509Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008613032Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"session_id":"f17e1ace-7488-4093-bac2-78b7ec985453"}
3592026-03-31T18:37:03.509Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.00862196Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185}
3602026-03-31T18:37:03.509Z{"msg":"Activated sub_volume 44516ed0-5555-41a8-a055-23b4a503eb2c","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008634409Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185}
3612026-03-31T18:37:03.510Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008642418Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185}
3622026-03-31T18:37:03.510Z{"msg":"44516ed0-5555-41a8-a055-23b4a503eb2c active request set","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008654666Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3632026-03-31T18:37:03.510Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008663665Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"client":"0","":"downstairs","session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3642026-03-31T18:37:03.510Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008673343Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"client":"1","":"downstairs","session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3652026-03-31T18:37:03.510Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008687221Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"client":"2","":"downstairs","session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3662026-03-31T18:37:03.510Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.00869673Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3672026-03-31T18:37:03.510Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008738355Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3682026-03-31T18:37:03.510Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008753033Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"io task","client":"1","":"downstairs","session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3692026-03-31T18:37:03.510Z{"msg":"ds_connection connected from Ok(127.0.0.1:46895)","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008767381Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3702026-03-31T18:37:03.510Z{"msg":"ds_connection connected from Ok(127.0.0.1:45450)","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008777609Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"io task","client":"1","":"downstairs","session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3712026-03-31T18:37:03.510Z{"msg":"ds_connection connected from Ok(127.0.0.1:63868)","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.008787858Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3722026-03-31T18:37:03.510Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.009202632Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"client":"1","":"downstairs","session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3732026-03-31T18:37:03.510Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.009225009Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"client":"0","":"downstairs","session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3742026-03-31T18:37:03.510Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.009257334Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"client":"2","":"downstairs","session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3752026-03-31T18:37:03.510Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.009403904Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3762026-03-31T18:37:03.510Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.009478014Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3772026-03-31T18:37:03.510Z{"msg":"[0]R flush_numbers: [5322, 5322, 5322, 5322, 5322]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.009495722Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"downstairs","":"downstairs","session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3782026-03-31T18:37:03.510Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.00951428Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"downstairs","":"downstairs","session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3792026-03-31T18:37:03.510Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.009528768Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"downstairs","":"downstairs","session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3802026-03-31T18:37:03.510Z{"msg":"[1]R flush_numbers: [5322, 5322, 5322, 5322, 5322]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.009539146Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"downstairs","":"downstairs","session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3812026-03-31T18:37:03.510Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.009553284Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"downstairs","":"downstairs","session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3822026-03-31T18:37:03.510Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.009563163Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"downstairs","":"downstairs","session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3832026-03-31T18:37:03.510Z{"msg":"[2]R flush_numbers: [5322, 5322, 5322, 5322, 5322]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.009573352Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"downstairs","":"downstairs","session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3842026-03-31T18:37:03.510Z{"msg":"Wait for a query_work_queue command to finish before sending IO
3852026-03-31T18:37:03.510Z[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.009591829Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"downstairs","":"downstairs","session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3862026-03-31T18:37:03.510Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.009601708Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"downstairs","":"downstairs","session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3872026-03-31T18:37:03.510Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.009614956Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"downstairs","":"downstairs","session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3882026-03-31T18:37:03.510Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.009624585Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"downstairs","":"downstairs","session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3892026-03-31T18:37:03.510Z{"msg":"Next flush: 5323","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.009634343Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"downstairs","":"downstairs","session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3902026-03-31T18:37:03.510Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.009644232Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"":"downstairs","":"downstairs","session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3912026-03-31T18:37:03.510Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.00965819Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3922026-03-31T18:37:03.510Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.009666839Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"client":"0","":"downstairs","session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3932026-03-31T18:37:03.510Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.009680627Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"client":"1","":"downstairs","session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3942026-03-31T18:37:03.510Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.009690146Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"client":"2","":"downstairs","session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3952026-03-31T18:37:03.510Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.009707573Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3962026-03-31T18:37:03.510Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.009716382Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3972026-03-31T18:37:03.510Z{"msg":"44516ed0-5555-41a8-a055-23b4a503eb2c is now active with session: ca701710-b404-4f91-8128-fae25eaa6497","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.00973044Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3982026-03-31T18:37:03.510Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.009739389Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185,"session_id":"ca701710-b404-4f91-8128-fae25eaa6497"}
3992026-03-31T18:37:03.511Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.009752667Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185}
4002026-03-31T18:37:03.511Z{"msg":"Activated sub_volume 44516ed0-5555-41a8-a055-23b4a503eb2c","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:02.009760886Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185}
4012026-03-31T18:37:03.511ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
4022026-03-31T18:37:03.511ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4032026-03-31T18:37:03.511ZFill test
4042026-03-31T18:37:03.531ZRead and Verify all blocks (0..100 range:false)
4052026-03-31T18:37:03.577ZWrote out file "/var/tmp/test_up-build/verify_file"
4062026-03-31T18:37:03.577ZCLIENT: Tests done. All submitted work has been ACK'd
4072026-03-31T18:37:03.577Z----------------------------------------------------------------
4082026-03-31T18:37:03.581Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4092026-03-31T18:37:03.581ZStates: Active Active Active
4102026-03-31T18:37:03.581ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4112026-03-31T18:37:03.581Z 1002 Acked Read 50 Done Done Done false
4122026-03-31T18:37:03.581Z STATES DS:0 DS:1 DS:2 TOTAL
4132026-03-31T18:37:03.581Z Sent 0 0 0 0
4142026-03-31T18:37:03.581Z Done 1 1 1 3
4152026-03-31T18:37:03.581Z Skipped 0 0 0 0
4162026-03-31T18:37:03.581Z Error 0 0 0 0
4172026-03-31T18:37:03.581ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4182026-03-31T18:37:03.581ZDownstairs last five completed: 1001 1000
4192026-03-31T18:37:03.581ZUpstairs last five completed: 1002 1001 1000
4202026-03-31T18:37:03.582Z----------------------------------------------------------------
4212026-03-31T18:37:03.582Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4222026-03-31T18:37:03.582ZStates: Active Active Active
4232026-03-31T18:37:03.582ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4242026-03-31T18:37:03.582Z 1002 Acked Read 50 Done Done Done false
4252026-03-31T18:37:03.582Z STATES DS:0 DS:1 DS:2 TOTAL
4262026-03-31T18:37:03.582Z Sent 0 0 0 0
4272026-03-31T18:37:03.582Z Done 1 1 1 3
4282026-03-31T18:37:03.582Z Skipped 0 0 0 0
4292026-03-31T18:37:03.582Z Error 0 0 0 0
4302026-03-31T18:37:03.582ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4312026-03-31T18:37:03.582ZDownstairs last five completed: 1001 1000
4322026-03-31T18:37:03.582ZUpstairs last five completed: 1002 1001 1000
4332026-03-31T18:37:03.582ZCLIENT: Up:0 ds:2 act:6
4342026-03-31T18:37:07.580Z----------------------------------------------------------------
4352026-03-31T18:37:07.580Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4362026-03-31T18:37:07.580ZDownstairs last five completed: 1003 1002 1001 1000
4372026-03-31T18:37:07.580ZUpstairs last five completed: 1003 1002 1001 1000
4382026-03-31T18:37:07.580Z----------------------------------------------------------------
4392026-03-31T18:37:07.580Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4402026-03-31T18:37:07.580ZDownstairs last five completed: 1003 1002 1001 1000
4412026-03-31T18:37:07.580ZUpstairs last five completed: 1003 1002 1001 1000
4422026-03-31T18:37:07.583ZCLIENT: Up:0 ds:0 act:6
4432026-03-31T18:37:07.583ZCLIENT: All crucible jobs finished, exiting program
4442026-03-31T18:37:07.583Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-31T18:37:06.084826437Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1185}
4452026-03-31T18:37:07.587ZRepair part 1 passed
4462026-03-31T18:37:07.587Z
4472026-03-31T18:37:07.587ZKill the current downstairs
4482026-03-31T18:37:07.631ZDownstairs 2 stopped
4492026-03-31T18:37:07.631Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
4502026-03-31T18:37:07.634ZNow put back the original so we have a mismatch
4512026-03-31T18:37:07.634Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
4522026-03-31T18:37:07.634ZRestart downstairs with old directory
4532026-03-31T18:37:07.646Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4542026-03-31T18:37:07.659ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4552026-03-31T18:37:07.663Z 0 000-009 32 32 31 4005 4005 4004 F F F <---
4562026-03-31T18:37:07.663Z 1 010-019 32 32 31 4005 4005 4004 F F F <---
4572026-03-31T18:37:07.663Z 2 020-029 32 32 31 4005 4005 4004 F F F <---
4582026-03-31T18:37:07.663Z 3 030-039 32 32 31 4005 4005 4004 F F F <---
4592026-03-31T18:37:07.663Z 4 040-049 32 32 31 4005 4005 4004 F F F <---
4602026-03-31T18:37:07.663ZMax gen: 32, Max flush: 4005
4612026-03-31T18:37:07.663ZError: Difference in extent metadata found!
4622026-03-31T18:37:07.663Zdump test found error as expected
4632026-03-31T18:37:07.663Z
4642026-03-31T18:37:07.663Z
4652026-03-31T18:37:07.663Z/var/tmp/bins/crutest verify --range -g 33 -q --key tE5JraOhcLCGMrtLvW4ovkqCD3iISXGWjE77AlqaQjk= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
4662026-03-31T18:37:07.678Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.183435442Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"task":"crutest"}
4672026-03-31T18:37:07.683Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.183767067Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"task":"crutest"}
4682026-03-31T18:37:07.683Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.184388453Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"task":"crutest"}
4692026-03-31T18:37:07.683Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.184647118Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193}
4702026-03-31T18:37:07.683Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"15d1a4bfe3549686eb976ff56710f1466c0e0e4a\",\n git_commit_timestamp: \"2026-03-31T18:13:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.184674394Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193}
4712026-03-31T18:37:07.683Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.184683113Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193}
4722026-03-31T18:37:07.683Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.184695331Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193}
4732026-03-31T18:37:07.683Z{"msg":"Crucible f042e7a7-0138-40e2-a925-b1ad4445a09a has session id: 024a7eeb-b6fc-4c12-942f-6c9ca00fa761","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.185100956Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
4742026-03-31T18:37:07.683Z{"msg":"Upstairs opts: Upstairs UUID: f042e7a7-0138-40e2-a925-b1ad4445a09a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.185154299Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
4752026-03-31T18:37:07.683Z{"msg":"Crucible stats registered with UUID: f042e7a7-0138-40e2-a925-b1ad4445a09a","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.185188495Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
4762026-03-31T18:37:07.683Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-31T18:37:06.185199973Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
4772026-03-31T18:37:07.683Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.185308228Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
4782026-03-31T18:37:07.684Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.185347923Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
4792026-03-31T18:37:07.684Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.185358982Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
4802026-03-31T18:37:07.684Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.186242572Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"task":"crutest"}
4812026-03-31T18:37:07.684Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.186294315Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193}
4822026-03-31T18:37:07.684Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"15d1a4bfe3549686eb976ff56710f1466c0e0e4a\",\n git_commit_timestamp: \"2026-03-31T18:13:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.186319032Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193}
4832026-03-31T18:37:07.684Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.18633199Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193}
4842026-03-31T18:37:07.684Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.186344068Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193}
4852026-03-31T18:37:07.684Z{"msg":"Crucible f042e7a7-0138-40e2-a925-b1ad4445a09a has session id: 4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.186528743Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
4862026-03-31T18:37:07.684Z{"msg":"Upstairs opts: Upstairs UUID: f042e7a7-0138-40e2-a925-b1ad4445a09a, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.18655162Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
4872026-03-31T18:37:07.684Z{"msg":"Crucible stats registered with UUID: f042e7a7-0138-40e2-a925-b1ad4445a09a","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.186569858Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
4882026-03-31T18:37:07.684Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-31T18:37:06.186579556Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
4892026-03-31T18:37:07.684Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.186596284Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
4902026-03-31T18:37:07.684Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.186607173Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
4912026-03-31T18:37:07.684Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.186633329Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
4922026-03-31T18:37:07.684Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.186785059Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"task":"crutest"}
4932026-03-31T18:37:07.688Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.186833042Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193}
4942026-03-31T18:37:07.688Z{"msg":"f042e7a7-0138-40e2-a925-b1ad4445a09a active request set","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.186853989Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
4952026-03-31T18:37:07.688Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.186864378Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"client":"0","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
4962026-03-31T18:37:07.688Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.186875196Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"client":"1","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
4972026-03-31T18:37:07.688Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.186885495Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"client":"2","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
4982026-03-31T18:37:07.688Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.186896134Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
4992026-03-31T18:37:07.688Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.186907032Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5002026-03-31T18:37:07.688Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.186917501Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5012026-03-31T18:37:07.688Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-03-31T18:37:06.187026906Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5022026-03-31T18:37:07.688Z{"msg":"ds_connection connected from Ok(127.0.0.1:41730)","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.1870704Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5032026-03-31T18:37:07.688Z{"msg":"ds_connection connected from Ok(127.0.0.1:49080)","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.187136101Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5042026-03-31T18:37:07.688Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-03-31T18:37:06.187159058Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5052026-03-31T18:37:07.688Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-03-31T18:37:06.187177155Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5062026-03-31T18:37:07.688Z{"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.187186694Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5072026-03-31T18:37:07.688Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.18721556Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5082026-03-31T18:37:07.688Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.18766389Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"client":"1","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5092026-03-31T18:37:07.688Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.187740819Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"client":"0","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5102026-03-31T18:37:07.688Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.187845175Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5112026-03-31T18:37:07.688Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:06.187871062Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5122026-03-31T18:37:17.686Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:16.187982247Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5132026-03-31T18:37:17.686Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-03-31T18:37:16.188044378Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5142026-03-31T18:37:17.686Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-03-31T18:37:16.188061536Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5152026-03-31T18:37:17.686Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-03-31T18:37:16.188078044Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5162026-03-31T18:37:17.690Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:16.188141305Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5172026-03-31T18:37:17.690Z{"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:16.188165002Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5182026-03-31T18:37:27.690Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.188732974Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5192026-03-31T18:37:27.691Z{"msg":"ds_connection connected from Ok(127.0.0.1:45982)","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.188855237Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5202026-03-31T18:37:27.699Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.190317659Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"client":"2","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5212026-03-31T18:37:27.699Z{"msg":"[0]R flush_numbers: [4005, 4005, 4005, 4005, 4005]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.190879963Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"downstairs","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5222026-03-31T18:37:27.699Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.190893772Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"downstairs","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5232026-03-31T18:37:27.699Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.19090424Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"downstairs","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5242026-03-31T18:37:27.700Z{"msg":"[1]R flush_numbers: [4005, 4005, 4005, 4005, 4005]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.190921268Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"downstairs","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5252026-03-31T18:37:27.700Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.190931676Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"downstairs","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5262026-03-31T18:37:27.700Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.190941545Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"downstairs","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5272026-03-31T18:37:27.700Z{"msg":"[2]R flush_numbers: [4004, 4004, 4004, 4004, 4004]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.190952254Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"downstairs","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5282026-03-31T18:37:27.700Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.190961882Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"downstairs","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5292026-03-31T18:37:27.700Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.190972081Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"downstairs","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5302026-03-31T18:37:27.700Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.19098155Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"downstairs","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5312026-03-31T18:37:27.700Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.190996718Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"downstairs","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5322026-03-31T18:37:27.700Z{"msg":"Next flush: 4006","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.191022504Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"downstairs","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5332026-03-31T18:37:27.700Z{"msg":"extent 0 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4004, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.191037892Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"mend","":"downstairs","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5342026-03-31T18:37:27.700Z{"msg":"extent:0 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4004, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.19105344Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"mend","":"downstairs","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5352026-03-31T18:37:27.700Z{"msg":"found dest for source 0 for extent at index 0 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.191064428Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"mend","":"downstairs","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5362026-03-31T18:37:27.700Z{"msg":"extent 1 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4004, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.191078956Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"mend","":"downstairs","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5372026-03-31T18:37:27.700Z{"msg":"extent:1 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4004, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.191093504Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"mend","":"downstairs","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5382026-03-31T18:37:27.700Z{"msg":"found dest for source 0 for extent at index 1 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.191104843Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"mend","":"downstairs","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5392026-03-31T18:37:27.700Z{"msg":"extent 2 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4004, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.191118781Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"mend","":"downstairs","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5402026-03-31T18:37:27.701Z{"msg":"extent:2 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4004, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.191132749Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"mend","":"downstairs","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5412026-03-31T18:37:27.701Z{"msg":"found dest for source 0 for extent at index 2 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.191143478Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"mend","":"downstairs","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5422026-03-31T18:37:27.701Z{"msg":"extent 3 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4004, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.191157556Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"mend","":"downstairs","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5432026-03-31T18:37:27.701Z{"msg":"extent:3 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4004, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.191172154Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"mend","":"downstairs","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5442026-03-31T18:37:27.701Z{"msg":"found dest for source 0 for extent at index 3 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.191182812Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"mend","":"downstairs","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5452026-03-31T18:37:27.701Z{"msg":"extent 4 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4004, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.191196701Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"mend","":"downstairs","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5462026-03-31T18:37:27.701Z{"msg":"extent:4 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4005, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4004, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.191210759Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"mend","":"downstairs","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5472026-03-31T18:37:27.701Z{"msg":"found dest for source 0 for extent at index 4 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.191221677Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"mend","":"downstairs","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5482026-03-31T18:37:27.701Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.191239165Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"client":"0","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5492026-03-31T18:37:27.701Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.191254033Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"client":"1","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5502026-03-31T18:37:27.701Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.191264411Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"client":"2","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5512026-03-31T18:37:27.701Z{"msg":"Full repair list: {ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.191285419Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"downstairs","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5522026-03-31T18:37:27.701Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 4006, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 4006, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 4006, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 4006, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 4006, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.191378846Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"downstairs","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5532026-03-31T18:37:27.701Z{"msg":"starting reconciliation f982ac9b-a9b6-460d-aa14-814d4cb57dd1: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.191390304Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"downstairs","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5542026-03-31T18:37:27.701Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.191664037Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"client":"2","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5552026-03-31T18:37:27.709Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.201117948Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"client":"2","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5562026-03-31T18:37:27.709Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.203337787Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"client":"2","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5572026-03-31T18:37:27.709Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.205194176Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"client":"2","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5582026-03-31T18:37:27.709Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.207172059Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"client":"2","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5592026-03-31T18:37:27.718Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.208918252Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"downstairs","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5602026-03-31T18:37:27.718Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.20893325Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"client":"0","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5612026-03-31T18:37:27.718Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.208943109Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"client":"1","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5622026-03-31T18:37:27.718Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.208957197Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"client":"2","":"downstairs","session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5632026-03-31T18:37:27.718Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.208966866Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5642026-03-31T18:37:27.718Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.208980484Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5652026-03-31T18:37:27.718Z{"msg":"f042e7a7-0138-40e2-a925-b1ad4445a09a is now active with session: 024a7eeb-b6fc-4c12-942f-6c9ca00fa761","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.208990692Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"session_id":"024a7eeb-b6fc-4c12-942f-6c9ca00fa761"}
5662026-03-31T18:37:27.719Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.209004011Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193}
5672026-03-31T18:37:27.719Z{"msg":"Activated sub_volume f042e7a7-0138-40e2-a925-b1ad4445a09a","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.20901236Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193}
5682026-03-31T18:37:27.719Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.209092159Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193}
5692026-03-31T18:37:27.719Z{"msg":"f042e7a7-0138-40e2-a925-b1ad4445a09a active request set","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.209105287Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
5702026-03-31T18:37:27.719Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.209114406Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"client":"0","":"downstairs","session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
5712026-03-31T18:37:27.719Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.209128524Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"client":"1","":"downstairs","session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
5722026-03-31T18:37:27.719Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.209148441Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"client":"2","":"downstairs","session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
5732026-03-31T18:37:27.719Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.20915802Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
5742026-03-31T18:37:27.719Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.209172568Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
5752026-03-31T18:37:27.719Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.209182996Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
5762026-03-31T18:37:27.719Z{"msg":"ds_connection connected from Ok(127.0.0.1:64592)","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.209193935Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
5772026-03-31T18:37:27.719Z{"msg":"ds_connection connected from Ok(127.0.0.1:43269)","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.209204614Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
5782026-03-31T18:37:27.719Z{"msg":"ds_connection connected from Ok(127.0.0.1:63554)","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.20923024Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
5792026-03-31T18:37:27.719Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.209648333Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"client":"1","":"downstairs","session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
5802026-03-31T18:37:27.719Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.20967352Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"client":"2","":"downstairs","session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
5812026-03-31T18:37:27.719Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.209686688Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"client":"0","":"downstairs","session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
5822026-03-31T18:37:27.720Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.209854036Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
5832026-03-31T18:37:27.720Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.209868644Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
5842026-03-31T18:37:27.720Z{"msg":"[0]R flush_numbers: [5323, 5323, 5323, 5323, 5323]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.209877992Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"downstairs","":"downstairs","session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
5852026-03-31T18:37:27.720Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.20989287Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"downstairs","":"downstairs","session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
5862026-03-31T18:37:27.720Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.209903019Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"downstairs","":"downstairs","session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
5872026-03-31T18:37:27.720Z{"msg":"[1]R flush_numbers: [5323, 5323, 5323, 5323, 5323]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.209917237Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"downstairs","":"downstairs","session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
5882026-03-31T18:37:27.720Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.209927326Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"downstairs","":"downstairs","session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
5892026-03-31T18:37:27.720Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.209941514Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"downstairs","":"downstairs","session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
5902026-03-31T18:37:27.720Z{"msg":"[2]R flush_numbers: [5323, 5323, 5323, 5323, 5323]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.209951622Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"downstairs","":"downstairs","session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
5912026-03-31T18:37:27.720Z{"msg":"[2]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.209965671Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"downstairs","":"downstairs","session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
5922026-03-31T18:37:27.720Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.209975749Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"downstairs","":"downstairs","session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
5932026-03-31T18:37:27.720Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.209989337Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"downstairs","":"downstairs","session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
5942026-03-31T18:37:27.720Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO
5952026-03-31T18:37:27.720Zlevel":30,"time":"2026-03-31T18:37:26.210003965Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"downstairs","":"downstairs","session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
5962026-03-31T18:37:27.721Z{"msg":"Next flush: 5324","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.210018093Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"downstairs","":"downstairs","session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
5972026-03-31T18:37:27.721Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.210027732Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"":"downstairs","":"downstairs","session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
5982026-03-31T18:37:27.721Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.21004197Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
5992026-03-31T18:37:27.721Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.210050989Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"client":"0","":"downstairs","session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
6002026-03-31T18:37:27.721Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.210064587Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"client":"1","":"downstairs","session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
6012026-03-31T18:37:27.721Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.210074106Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"client":"2","":"downstairs","session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
6022026-03-31T18:37:27.721Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.210088014Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
6032026-03-31T18:37:27.721Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.210097453Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
6042026-03-31T18:37:27.721Z{"msg":"f042e7a7-0138-40e2-a925-b1ad4445a09a is now active with session: 4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.210111681Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
6052026-03-31T18:37:27.721Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.21012045Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193,"session_id":"4495ba3e-3cf3-4dbf-a3cb-0807e0b6c84d"}
6062026-03-31T18:37:27.721Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.210133408Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193}
6072026-03-31T18:37:27.721Z{"msg":"Activated sub_volume f042e7a7-0138-40e2-a925-b1ad4445a09a","v":0,"name":"crucible","level":30,"time":"2026-03-31T18:37:26.210141467Z","hostname":"w-01KN2JNFQKF115ANF9FAMMJH5X","pid":1193}
6082026-03-31T18:37:27.722ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
6092026-03-31T18:37:27.722ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
6102026-03-31T18:37:27.722ZRead and Verify all blocks (0..100 range:true)
6112026-03-31T18:37:27.757ZWrote out file "/var/tmp/test_up-build/verify_file"
6122026-03-31T18:37:27.757ZVerify test completed
6132026-03-31T18:37:27.760ZWrote out file "/var/tmp/test_up-build/verify_file"
6142026-03-31T18:37:27.761ZCLIENT: Tests done. All submitted work has been ACK'd
6152026-03-31T18:37:27.761Z----------------------------------------------------------------
6162026-03-31T18:37:27.761Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6172026-03-31T18:37:27.761ZStates: Active Active Active
6182026-03-31T18:37:27.761ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6192026-03-31T18:37:27.761Z 1000 Acked Read 50 Done Done Done false
6202026-03-31T18:37:27.761Z STATES DS:0 DS:1 DS:2 TOTAL
6212026-03-31T18:37:27.761Z Sent 0 0 0 0
6222026-03-31T18:37:27.761Z Done 1 1 1 3
6232026-03-31T18:37:27.761Z Skipped 0 0 0 0
6242026-03-31T18:37:27.761Z Error 0 0 0 0
6252026-03-31T18:37:27.761ZLast Flush: None None None
6262026-03-31T18:37:27.761ZDownstairs last five completed:
6272026-03-31T18:37:27.761ZUpstairs last five completed: 1000
6282026-03-31T18:37:27.761Z----------------------------------------------------------------
6292026-03-31T18:37:27.761Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6302026-03-31T18:37:27.761ZStates: Active Active Active
6312026-03-31T18:37:27.761ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6322026-03-31T18:37:27.761Z 1000 Acked Read 50 Done Done Done false
6332026-03-31T18:37:27.761Z STATES DS:0 DS:1 DS:2 TOTAL
6342026-03-31T18:37:27.762Z Sent 0 0 0 0
6352026-03-31T18:37:27.762Z Done 1 1 1 3
6362026-03-31T18:37:27.762Z Skipped 0 0 0 0
6372026-03-31T18:37:27.762Z Error 0 0 0 0
6382026-03-31T18:37:27.762ZLast Flush: None None None
6392026-03-31T18:37:27.762ZDownstairs last five completed:
6402026-03-31T18:37:27.762ZUpstairs last five completed: 1000
6412026-03-31T18:37:27.762ZCLIENT: Up:0 ds:2 act:6
6422026-03-31T18:37:31.762Z----------------------------------------------------------------
6432026-03-31T18:37:31.762Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6442026-03-31T18:37:31.762ZDownstairs last five completed: 1001 1000
6452026-03-31T18:37:31.762ZUpstairs last five completed: 1001 1000
6462026-03-31T18:37:31.763Z----------------------------------------------------------------
6472026-03-31T18:37:31.763Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6482026-03-31T18:37:31.763ZDownstairs last five completed: 1001 1000
6492026-03-31T18:37:31.763ZUpstairs last five completed: 1001 1000
6502026-03-31T18:37:31.763ZCLIENT: Up:0 ds:0 act:6
6512026-03-31T18:37:31.763ZCLIENT: All crucible jobs finished, exiting program
6522026-03-31T18:37:31.767ZRepair part 2 passed
6532026-03-31T18:37:31.767Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
6542026-03-31T18:37:31.779ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6552026-03-31T18:37:31.779Z 0 000-009 32 32 32 4005 4005 4005 F F F
6562026-03-31T18:37:31.779Z 1 010-019 32 32 32 4005 4005 4005 F F F
6572026-03-31T18:37:31.779Z 2 020-029 32 32 32 4005 4005 4005 F F F
6582026-03-31T18:37:31.782Z 3 030-039 32 32 32 4005 4005 4005 F F F
6592026-03-31T18:37:31.782Z 4 040-049 32 32 32 4005 4005 4005 F F F
6602026-03-31T18:37:31.782ZMax gen: 32, Max flush: 4005
6612026-03-31T18:37:31.782Zdump test passed
6622026-03-31T18:37:31.782Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
6632026-03-31T18:37:31.793Z Extent 1
6642026-03-31T18:37:31.796ZGEN 32 32 32
6652026-03-31T18:37:31.796ZFLUSH_ID 4005 4005 4005
6662026-03-31T18:37:31.796ZDIRTY
6672026-03-31T18:37:31.796Z
6682026-03-31T18:37:31.796ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
6692026-03-31T18:37:31.796Z010 A A A A A A
6702026-03-31T18:37:31.796Z011 A A A A A A
6712026-03-31T18:37:31.796Z012 A A A A A A
6722026-03-31T18:37:31.796Z013 A A A A A A
6732026-03-31T18:37:31.797Z014 A A A A A A
6742026-03-31T18:37:31.800Z015 A A A A A A
6752026-03-31T18:37:31.800Z016 A A A A A A
6762026-03-31T18:37:31.800Z017 A A A A A A
6772026-03-31T18:37:31.800Z018 A A A A A A
6782026-03-31T18:37:31.800Z019 A A A A A A
6792026-03-31T18:37:31.803Zdump extent test passed
6802026-03-31T18:37:31.803Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
6812026-03-31T18:37:31.811ZExtent 2 Block in extent 0 Actual block 20
6822026-03-31T18:37:31.811Z
6832026-03-31T18:37:31.816Z DATA SHA256 VER
6842026-03-31T18:37:31.816Z------ ---------------------------------------------------------------- ---
6852026-03-31T18:37:31.816Z 0 5c266f72f53e0247e265a915d0a143650547867c73c6a50ed5815a2cf373e007 A
6862026-03-31T18:37:31.816Z 1 5c266f72f53e0247e265a915d0a143650547867c73c6a50ed5815a2cf373e007 A
6872026-03-31T18:37:31.816Z 2 5c266f72f53e0247e265a915d0a143650547867c73c6a50ed5815a2cf373e007 A
6882026-03-31T18:37:31.816Z
6892026-03-31T18:37:31.816ZNONCES 0 1 2 DIFF
6902026-03-31T18:37:31.816Z------ ------------------------ ------------------------ ------------------------ -----
6912026-03-31T18:37:31.816Z 0 4941788791f784d8a8e520c9 4941788791f784d8a8e520c9 4941788791f784d8a8e520c9
6922026-03-31T18:37:31.816Z
6932026-03-31T18:37:31.816Z TAGS 0 1 2 DIFF
6942026-03-31T18:37:31.816Z------ -------------------------------- -------------------------------- -------------------------------- -----
6952026-03-31T18:37:31.816Z 0 4624b68cd42fd528e06bc28962205218 4624b68cd42fd528e06bc28962205218 4624b68cd42fd528e06bc28962205218
6962026-03-31T18:37:31.816Z
6972026-03-31T18:37:31.816ZHASHES 0 1 2 DIFF
6982026-03-31T18:37:31.816Z------ ---------------- ---------------- ---------------- -----
6992026-03-31T18:37:31.816Z
7002026-03-31T18:37:31.816Zdump block test passed
7012026-03-31T18:37:31.816ZInitial upstairs tests have completed, stopping all downstairs
7022026-03-31T18:37:36.837ZCreating 7 larger downstairs regions
7032026-03-31T18:37:37.037ZStarting 7 downstairs
7042026-03-31T18:37:42.043Zdsc restarted at PID: 1209
7052026-03-31T18:37:42.144ZNow do the replace-reconcile test
7062026-03-31T18:37:42.156ZUsing 8870 for the replacement port
7072026-03-31T18:41:18.861ZCompleted test: replace-reconcile
7082026-03-31T18:41:18.874ZNow do the replace-before-active test
7092026-03-31T18:45:13.765ZCompleted test: replace-before-active
7102026-03-31T18:45:13.765ZAll tests have completed, stopping all downstairs
7112026-03-31T18:45:13.782Z
7122026-03-31T18:45:13.785ZAll Tests have passed
7132026-03-31T18:45:13.785Z11:52 Test duration
7142026-03-31T18:45:13.786Z
7152026-03-31T18:45:13.786Zreal 11:51.840545304
7162026-03-31T18:45:13.786Zuser 17:00.616508883
7172026-03-31T18:45:13.786Zsys 11.707067891
7182026-03-31T18:45:13.786Ztrap 0.413848427
7192026-03-31T18:45:13.786Ztflt 0.043456997
7202026-03-31T18:45:13.786Zdflt 0.015846647
7212026-03-31T18:45:13.786Zkflt 0.014058403
7222026-03-31T18:45:13.786Zlock 16:30:14.440992866
7232026-03-31T18:45:13.786Zslp 51:54.121406361
7242026-03-31T18:45:13.786Zlat 1:48.410039692
7252026-03-31T18:45:13.786Zstop 0.019437822
7262026-03-31T18:45:13.786Z+ echo 'test-up-2region-encrypted ends'
7272026-03-31T18:45:13.786Ztest-up-2region-encrypted ends
7282026-03-31T18:45:18.788Zprocess exited: duration 720408 ms, exit code 0
7292026-03-31T18:45:18.789Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
7302026-03-31T18:46:18.845Zexec warning: : stdout descriptor may be held open by a background process; giving up!
7312026-03-31T18:46:18.845Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
7322026-03-31T18:46:18.858Zfound 17 output files
7332026-03-31T18:46:18.869Zuploading: /tmp/test_up-build/dsc-out.txt (58390 bytes)
7342026-03-31T18:46:19.879Zuploaded: /tmp/test_up-build/dsc-out.txt
7352026-03-31T18:46:19.879Zuploading: /tmp/test_up-build/test_up_out.txt (23664171 bytes)
7362026-03-31T18:46:21.037Zuploaded: /tmp/test_up-build/test_up_out.txt
7372026-03-31T18:46:21.037Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (550936 bytes)
7382026-03-31T18:46:22.054Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
7392026-03-31T18:46:22.054Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (548320 bytes)
7402026-03-31T18:46:23.069Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
7412026-03-31T18:46:23.069Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (978663 bytes)
7422026-03-31T18:46:24.091Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
7432026-03-31T18:46:24.091Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (543028 bytes)
7442026-03-31T18:46:25.107Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
7452026-03-31T18:46:25.108Zuploading: /tmp/test_up-build/dsc/downstairs-8850.txt (1343655 bytes)
7462026-03-31T18:46:26.128Zuploaded: /tmp/test_up-build/dsc/downstairs-8850.txt
7472026-03-31T18:46:26.128Zuploading: /tmp/test_up-build/dsc/downstairs-8860.txt (47317 bytes)
7482026-03-31T18:46:27.139Zuploaded: /tmp/test_up-build/dsc/downstairs-8860.txt
7492026-03-31T18:46:27.139Zuploading: /tmp/test_up-build/dsc/downstairs-8870.txt (3602451 bytes)
7502026-03-31T18:46:28.172Zuploaded: /tmp/test_up-build/dsc/downstairs-8870.txt
7512026-03-31T18:46:28.172Zuploading: /tmp/debug/df.txt (1270 bytes)
7522026-03-31T18:46:29.182Zuploaded: /tmp/debug/df.txt
7532026-03-31T18:46:29.182Zuploading: /tmp/debug/dtrace.txt (1018112 bytes)
7542026-03-31T18:46:29.201Zupload warning: file "/tmp/debug/dtrace.txt" changed size mid upload: 1018112 -> 1155971
7552026-03-31T18:46:30.208Zuploaded: /tmp/debug/dtrace.txt
7562026-03-31T18:46:30.208Zuploading: /tmp/debug/iostat.txt (231810 bytes)
7572026-03-31T18:46:30.217Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 231810 -> 235386
7582026-03-31T18:46:31.223Zuploaded: /tmp/debug/iostat.txt
7592026-03-31T18:46:31.223Zuploading: /tmp/debug/mpstat.txt (565070 bytes)
7602026-03-31T18:46:31.235Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 565070 -> 574521
7612026-03-31T18:46:32.246Zuploaded: /tmp/debug/mpstat.txt
7622026-03-31T18:46:32.246Zuploading: /tmp/debug/paging.txt (100125 bytes)
7632026-03-31T18:46:32.254Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 100125 -> 102023
7642026-03-31T18:46:33.261Zuploaded: /tmp/debug/paging.txt
7652026-03-31T18:46:33.261Zuploading: /tmp/debug/prstat.txt (1100274 bytes)
7662026-03-31T18:46:33.278Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 1100274 -> 1118417
7672026-03-31T18:46:34.284Zuploaded: /tmp/debug/prstat.txt
7682026-03-31T18:46:34.284Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
7692026-03-31T18:46:35.292Zuploaded: /tmp/debug/psrinfo.txt
7702026-03-31T18:46:35.292Zuploading: /tmp/debug/upstairs-info.txt (168028 bytes)
7712026-03-31T18:46:36.305Zuploaded: /tmp/debug/upstairs-info.txt