01KRZ990HJXRVS9XHKS65SV2N1: test-up-2region-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01KRZ99E5SK2RNWF410590AZJ9

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12026-05-19T05:06:03.914Zjob dependencies complete; ready to run (waiting for 12 m 49 s)
22026-05-19T05:07:05.157Zjob assigned to worker 01KRZA15QNXN04FPEYN6BYX5W8 [factory aws, i-0861c94e5739fec33] (queued for 1 m 1 s)
32026-05-19T05:07:11.296Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42026-05-19T05:07:12.933Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52026-05-19T05:07:12.941Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62026-05-19T05:07:14.494Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72026-05-19T05:07:14.498Zdownloading input: /input/build/work/bins/crutest.gz
82026-05-19T05:07:16.142Zdownloaded input: /input/build/work/bins/crutest.gz
92026-05-19T05:07:16.145Zdownloading input: /input/build/work/bins/dsc.gz
102026-05-19T05:07:16.885Zdownloaded input: /input/build/work/bins/dsc.gz
112026-05-19T05:07:16.895Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122026-05-19T05:07:16.916Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132026-05-19T05:07:16.920Zdownloading input: /input/build/work/scripts/test_ds.sh
142026-05-19T05:07:16.948Zdownloaded input: /input/build/work/scripts/test_ds.sh
152026-05-19T05:07:16.952Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162026-05-19T05:07:16.974Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172026-05-19T05:07:16.974Zdownloading input: /input/build/work/scripts/test_repair.sh
182026-05-19T05:07:17.004Zdownloaded input: /input/build/work/scripts/test_repair.sh
192026-05-19T05:07:17.004Zdownloading input: /input/build/work/scripts/test_replay.sh
202026-05-19T05:07:17.036Zdownloaded input: /input/build/work/scripts/test_replay.sh
212026-05-19T05:07:17.036Zdownloading input: /input/build/work/scripts/test_up.sh
222026-05-19T05:07:17.061Zdownloaded input: /input/build/work/scripts/test_up.sh
232026-05-19T05:07:17.064Zdownloading input: /input/build/work/scripts/upstairs_info.d
242026-05-19T05:07:17.091Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252026-05-19T05:07:17.095Zdownloading input: /input/build/tmp/cargo-test-out.log
262026-05-19T05:07:17.625Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272026-05-19T05:07:17.625Zstarting task 0: "setup"
282026-05-19T05:07:17.631Z++ uname -s
292026-05-19T05:07:17.634Z+ kern=SunOS
302026-05-19T05:07:17.634Z+ build_user=build
312026-05-19T05:07:17.634Z+ build_uid=12345
322026-05-19T05:07:17.634Z+ work_dir=/work
332026-05-19T05:07:17.634Z+ input_dir=/input
342026-05-19T05:07:17.634Z+ [[ 0 == 12345 ]]
352026-05-19T05:07:17.634Z+ case "$kern" in
362026-05-19T05:07:17.634Z+ groupadd -g 12345 build
372026-05-19T05:07:17.638Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382026-05-19T05:07:19.642Z+ zfs create -o mountpoint=/work rpool/work
392026-05-19T05:07:20.069Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402026-05-19T05:07:20.072Z+ home_fs=zfs
412026-05-19T05:07:20.072Z+ [[ zfs == autofs ]]
422026-05-19T05:07:20.072Z+ mkdir -p /home/build
432026-05-19T05:07:20.072Z+ chown build:build /home/build /work
442026-05-19T05:07:22.074Z+ chmod 0700 /home/build /work
452026-05-19T05:07:22.077Zprocess exited: duration 4451 ms, exit code 0
 
462026-05-19T05:07:22.083Zstarting task 1: "authentication"
472026-05-19T05:07:22.100Zprocess exited: duration 17 ms, exit code 0
 
482026-05-19T05:07:22.106Zstarting task 2: "build"
492026-05-19T05:07:22.109Z+ banner cores
502026-05-19T05:07:22.112Z
512026-05-19T05:07:22.112Z #### #### ##### ###### ####
522026-05-19T05:07:22.112Z # # # # # # # #
532026-05-19T05:07:22.112Z # # # # # ##### ####
542026-05-19T05:07:22.112Z # # # ##### # #
552026-05-19T05:07:22.112Z # # # # # # # # #
562026-05-19T05:07:22.113Z #### #### # # ###### ####
572026-05-19T05:07:22.113Z
582026-05-19T05:07:22.113Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
592026-05-19T05:07:22.117Z+ banner unpack
602026-05-19T05:07:22.120Z
612026-05-19T05:07:22.120Z # # # # ##### ## #### # #
622026-05-19T05:07:22.120Z # # ## # # # # # # # # #
632026-05-19T05:07:22.120Z # # # # # # # # # # ####
642026-05-19T05:07:22.120Z # # # # # ##### ###### # # #
652026-05-19T05:07:22.120Z # # # ## # # # # # # #
662026-05-19T05:07:22.120Z #### # # # # # #### # #
672026-05-19T05:07:22.120Z
682026-05-19T05:07:22.120Z+ mkdir -p /var/tmp/bins
692026-05-19T05:07:22.120Z+ for t in "$input/bins/"*.gz
702026-05-19T05:07:22.123Z++ basename /input/build/work/bins/crucible-downstairs.gz
712026-05-19T05:07:22.123Z+ b=crucible-downstairs.gz
722026-05-19T05:07:22.123Z+ b=crucible-downstairs
732026-05-19T05:07:22.123Z+ gunzip
742026-05-19T05:07:24.801Z+ chmod +x /var/tmp/bins/crucible-downstairs
752026-05-19T05:07:24.804Z+ for t in "$input/bins/"*.gz
762026-05-19T05:07:24.804Z++ basename /input/build/work/bins/crucible-hammer.gz
772026-05-19T05:07:24.806Z+ b=crucible-hammer.gz
782026-05-19T05:07:24.806Z+ b=crucible-hammer
792026-05-19T05:07:24.806Z+ gunzip
802026-05-19T05:07:27.308Z+ chmod +x /var/tmp/bins/crucible-hammer
812026-05-19T05:07:27.311Z+ for t in "$input/bins/"*.gz
822026-05-19T05:07:27.311Z++ basename /input/build/work/bins/crutest.gz
832026-05-19T05:07:27.314Z+ b=crutest.gz
842026-05-19T05:07:27.314Z+ b=crutest
852026-05-19T05:07:27.314Z+ gunzip
862026-05-19T05:07:29.948Z+ chmod +x /var/tmp/bins/crutest
872026-05-19T05:07:29.951Z+ for t in "$input/bins/"*.gz
882026-05-19T05:07:29.951Z++ basename /input/build/work/bins/dsc.gz
892026-05-19T05:07:29.954Z+ b=dsc.gz
902026-05-19T05:07:29.954Z+ b=dsc
912026-05-19T05:07:29.954Z+ gunzip
922026-05-19T05:07:31.160Z+ chmod +x /var/tmp/bins/dsc
932026-05-19T05:07:31.163Z+ export BINDIR=/var/tmp/bins
942026-05-19T05:07:31.163Z+ BINDIR=/var/tmp/bins
952026-05-19T05:07:31.163Z+ jobpid=1091
962026-05-19T05:07:31.163Z+ echo 'Setup debug logging'
972026-05-19T05:07:31.163Z+ mkdir /tmp/debug
982026-05-19T05:07:31.163ZSetup debug logging
992026-05-19T05:07:31.163Z+ sleep 3600
1002026-05-19T05:07:31.166Z+ psrinfo -v
1012026-05-19T05:07:31.169Z+ df -h
1022026-05-19T05:07:31.173Z+ prstat -d d -mLc 1
1032026-05-19T05:07:31.176Z+ iostat -T d -xn 1
1042026-05-19T05:07:31.176Z+ mpstat -T d 1
1052026-05-19T05:07:31.176Z+ vmstat -T d -p 1
1062026-05-19T05:07:31.176Z+ banner test_up_2r_encrypted
1072026-05-19T05:07:31.176Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1082026-05-19T05:07:31.179Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1092026-05-19T05:07:31.184Z #####
1102026-05-19T05:07:31.184Z ##### ###### #### ##### # # ##### # # #####
1112026-05-19T05:07:31.184Z # # # # # # # # # # #
1122026-05-19T05:07:31.184Z # ##### #### # # # # # ##### # #
1132026-05-19T05:07:31.184Z # # # # # # ##### # #####
1142026-05-19T05:07:31.184Z # # # # # # # # # # #
1152026-05-19T05:07:31.184Z # ###### #### # ####### #### # ####### ####### # #
1162026-05-19T05:07:31.184Z
1172026-05-19T05:07:31.184Z+ ptime -m bash /input/build/work/scripts/test_up.sh -r 2 -N encrypted
1182026-05-19T05:07:31.222Z/input/build/work
1192026-05-19T05:07:31.230ZUsing 2 region sets
1202026-05-19T05:07:31.230ZTurn off color for downstairs dump
1212026-05-19T05:07:31.244ZUpstairs using key: RqBeYhUu4wg0w+PD4TB5syu41lzm6qyNsK0V6PdJSVE=
1222026-05-19T05:07:31.247Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1232026-05-19T05:07:31.247ZCreating 6 downstairs regions
1242026-05-19T05:07:31.463ZStarting 6 downstairs
1252026-05-19T05:07:31.466Zdsc started at PID: 1139
1262026-05-19T05:07:36.557ZDisable automatic restart on all downstairs
1272026-05-19T05:07:36.569Z
1282026-05-19T05:07:36.569ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1292026-05-19T05:07:36.573ZRunning test: span
1302026-05-19T05:07:40.684ZCompleted test: span
1312026-05-19T05:07:45.694ZRunning test: big
1322026-05-19T05:07:50.005ZCompleted test: big
1332026-05-19T05:07:54.998ZRunning test: dep
1342026-05-19T05:08:14.392ZCompleted test: dep
1352026-05-19T05:08:19.399ZRunning test: balloon
1362026-05-19T05:10:44.125ZCompleted test: balloon
1372026-05-19T05:10:49.132ZRunning test: deactivate
1382026-05-19T05:10:53.456ZCompleted test: deactivate
1392026-05-19T05:10:58.463ZRunning hammer
1402026-05-19T05:11:10.659ZRun repair tests
1412026-05-19T05:11:10.662Z/var/tmp/bins/crutest fill -g 31 -q --key RqBeYhUu4wg0w+PD4TB5syu41lzm6qyNsK0V6PdJSVE= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1422026-05-19T05:11:10.682Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.261875218Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"task":"crutest"}
1432026-05-19T05:11:10.685Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.262193289Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"task":"crutest"}
1442026-05-19T05:11:10.685Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.262922718Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"task":"crutest"}
1452026-05-19T05:11:10.685Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.263174323Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182}
1462026-05-19T05:11:10.685Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4196b24f875a2f584277e37760a6e226abbc3bf5\",\n git_commit_timestamp: \"2026-05-19T04:52:46.000000000Z\",\n git_branch: \"renovate/tracing-subscriber-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.263220341Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182}
1472026-05-19T05:11:10.685Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.26323417Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182}
1482026-05-19T05:11:10.685Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.263246629Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182}
1492026-05-19T05:11:10.685Z{"msg":"Crucible 7cb60a63-9787-4f59-8d59-b2f624b53263 has session id: c5804a0c-1b2b-480c-a42d-2f9783c08ba1","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.263698533Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1502026-05-19T05:11:10.685Z{"msg":"Upstairs opts: Upstairs UUID: 7cb60a63-9787-4f59-8d59-b2f624b53263, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.263722982Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1512026-05-19T05:11:10.685Z{"msg":"Crucible stats registered with UUID: 7cb60a63-9787-4f59-8d59-b2f624b53263","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.263737371Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1522026-05-19T05:11:10.685Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-19T05:11:09.263746741Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1532026-05-19T05:11:10.686Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.263904412Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1542026-05-19T05:11:10.686Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.26394208Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1552026-05-19T05:11:10.686Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.263953859Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1562026-05-19T05:11:10.686Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.264784821Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"task":"crutest"}
1572026-05-19T05:11:10.686Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.264845848Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182}
1582026-05-19T05:11:10.686Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4196b24f875a2f584277e37760a6e226abbc3bf5\",\n git_commit_timestamp: \"2026-05-19T04:52:46.000000000Z\",\n git_branch: \"renovate/tracing-subscriber-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.264871027Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182}
1592026-05-19T05:11:10.686Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.264883836Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182}
1602026-05-19T05:11:10.686Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.264895815Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182}
1612026-05-19T05:11:10.686Z{"msg":"Crucible 7cb60a63-9787-4f59-8d59-b2f624b53263 has session id: 79fee139-4360-425d-aae5-649d98242bf9","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.265068425Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
1622026-05-19T05:11:10.686Z{"msg":"Upstairs opts: Upstairs UUID: 7cb60a63-9787-4f59-8d59-b2f624b53263, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.265088174Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
1632026-05-19T05:11:10.686Z{"msg":"Crucible stats registered with UUID: 7cb60a63-9787-4f59-8d59-b2f624b53263","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.265098074Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
1642026-05-19T05:11:10.686Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-19T05:11:09.265107433Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
1652026-05-19T05:11:10.686Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.26515303Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
1662026-05-19T05:11:10.686Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.2651681Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
1672026-05-19T05:11:10.686Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.265178449Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
1682026-05-19T05:11:10.686Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.265301492Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"task":"crutest"}
1692026-05-19T05:11:10.686Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.265318591Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182}
1702026-05-19T05:11:10.686Z{"msg":"7cb60a63-9787-4f59-8d59-b2f624b53263 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.265351879Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1712026-05-19T05:11:10.686Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.265366008Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"client":"0","":"downstairs","session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1722026-05-19T05:11:10.686Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.265376388Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"client":"1","":"downstairs","session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1732026-05-19T05:11:10.686Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.265393577Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"client":"2","":"downstairs","session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1742026-05-19T05:11:10.686Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.265403446Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1752026-05-19T05:11:10.686Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.265413706Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1762026-05-19T05:11:10.686Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.265441404Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1772026-05-19T05:11:10.686Z{"msg":"ds_connection connected from Ok(127.0.0.1:60063)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.265455883Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1782026-05-19T05:11:10.686Z{"msg":"ds_connection connected from Ok(127.0.0.1:50436)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.26550799Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1792026-05-19T05:11:10.686Z{"msg":"ds_connection connected from Ok(127.0.0.1:46996)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.265544778Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1802026-05-19T05:11:10.690Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266228209Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"client":"1","":"downstairs","session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1812026-05-19T05:11:10.690Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266265997Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"client":"0","":"downstairs","session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1822026-05-19T05:11:10.690Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266278016Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"client":"2","":"downstairs","session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1832026-05-19T05:11:10.690Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266443017Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1842026-05-19T05:11:10.690Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266456986Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1852026-05-19T05:11:10.690Z{"msg":"[0]R flush_numbers: [3999, 3999, 3999, 3999, 3999]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266471405Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"downstairs","":"downstairs","session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1862026-05-19T05:11:10.690Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266485864Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"downstairs","":"downstairs","session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1872026-05-19T05:11:10.691Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266496074Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"downstairs","":"downstairs","session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1882026-05-19T05:11:10.691Z{"msg":"[1]R flush_numbers: [3999, 3999, 3999, 3999, 3999]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266506343Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"downstairs","":"downstairs","session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1892026-05-19T05:11:10.691Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266520432Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"downstairs","":"downstairs","session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1902026-05-19T05:11:10.691Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266530952Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"downstairs","":"downstairs","session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1912026-05-19T05:11:10.691Z{"msg":"[2]R flush_numbers: [3999, 3999, 3999, 3999, 3999]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.26656367Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"downstairs","":"downstairs","session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1922026-05-19T05:11:10.691Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266593318Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"downstairs","":"downstairs","session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1932026-05-19T05:11:10.691Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266607677Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"downstairs","":"downstairs","session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1942026-05-19T05:11:10.691Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266621287Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"downstairs","":"downstairs","session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1952026-05-19T05:11:10.691Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266631076Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"downstairs","":"downstairs","session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1962026-05-19T05:11:10.691Z{"msg":"Next flush: 4000","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266640855Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"downstairs","":"downstairs","session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1972026-05-19T05:11:10.691Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266654565Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"downstairs","":"downstairs","session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1982026-05-19T05:11:10.691Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266664914Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
1992026-05-19T05:11:10.691Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266678113Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"client":"0","":"downstairs","session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
2002026-05-19T05:11:10.691Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266688093Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"client":"1","":"downstairs","session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
2012026-05-19T05:11:10.691Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266697692Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"client":"2","":"downstairs","session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
2022026-05-19T05:11:10.691Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266711721Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
2032026-05-19T05:11:10.691Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266721021Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
2042026-05-19T05:11:10.691Z{"msg":"7cb60a63-9787-4f59-8d59-b2f624b53263 is now active with session: c5804a0c-1b2b-480c-a42d-2f9783c08ba1","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.26673133Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
2052026-05-19T05:11:10.691Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.26674055Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"session_id":"c5804a0c-1b2b-480c-a42d-2f9783c08ba1"}
2062026-05-19T05:11:10.691Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266750039Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182}
2072026-05-19T05:11:10.691Z{"msg":"Activated sub_volume 7cb60a63-9787-4f59-8d59-b2f624b53263","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266762568Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182}
2082026-05-19T05:11:10.691Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266770558Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182}
2092026-05-19T05:11:10.691Z{"msg":"7cb60a63-9787-4f59-8d59-b2f624b53263 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266779108Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2102026-05-19T05:11:10.691Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266791817Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"client":"0","":"downstairs","session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2112026-05-19T05:11:10.691Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266801446Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"client":"1","":"downstairs","session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2122026-05-19T05:11:10.691Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266811176Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"client":"2","":"downstairs","session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2132026-05-19T05:11:10.691Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266871382Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2142026-05-19T05:11:10.691Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266940348Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2152026-05-19T05:11:10.691Z{"msg":"ds_connection connected from Ok(127.0.0.1:52131)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266965357Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2162026-05-19T05:11:10.691Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266975886Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2172026-05-19T05:11:10.691Z{"msg":"ds_connection connected from Ok(127.0.0.1:47827)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.266994245Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2182026-05-19T05:11:10.691Z{"msg":"ds_connection connected from Ok(127.0.0.1:49935)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.267004645Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2192026-05-19T05:11:10.691Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.267225862Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"client":"0","":"downstairs","session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2202026-05-19T05:11:10.691Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.26726879Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"client":"2","":"downstairs","session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2212026-05-19T05:11:10.691Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.267284539Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"client":"1","":"downstairs","session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2222026-05-19T05:11:10.691Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.267365444Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2232026-05-19T05:11:10.692Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.267388713Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2242026-05-19T05:11:10.692Z{"msg":"[0]R flush_numbers: [5319, 5319, 5322, 5321, 5309]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.267473878Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"downstairs","":"downstairs","session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2252026-05-19T05:11:10.692Z{"msg":"[0]R generation: [13, 13, 14, 14, 11]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.267485267Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"downstairs","":"downstairs","session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2262026-05-19T05:11:10.692Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.267495897Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"downstairs","":"downstairs","session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2272026-05-19T05:11:10.692Z{"msg":"[1]R flush_numbers: [5319, 5319, 5322, 5321, 5309]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.267506746Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"downstairs","":"downstairs","session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2282026-05-19T05:11:10.692Z{"msg":"[1]R generation: [13, 13, 14, 14, 11]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.267517255Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"downstairs","":"downstairs","session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2292026-05-19T05:11:10.692Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.267527685Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"downstairs","":"downstairs","session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2302026-05-19T05:11:10.692Z{"msg":"[2]R flush_numbers: [5319, 5319, 5322, 5321, 5309]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.267538054Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"downstairs","":"downstairs","session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2312026-05-19T05:11:10.692Z{"msg":"[2]R generation: [13, 13, 14, 14, 11]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.267556013Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"downstairs","":"downstairs","session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2322026-05-19T05:11:10.692Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.267566453Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"downstairs","":"downstairs","session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2332026-05-19T05:11:10.692Z{"msg":"Max found gen is 15","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.267576502Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"downstairs","":"downstairs","session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2342026-05-19T05:11:10.692Z{"msg":"Generation requested: 31 >= found:15","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.267590341Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"downstairs","":"downstairs","session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2352026-05-19T05:11:10.692Z{"msg":"Next flush: 5323","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.267599931Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"downstairs","":"downstairs","session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2362026-05-19T05:11:10.692Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.26761396Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"":"downstairs","":"downstairs","session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2372026-05-19T05:11:10.692Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.267624479Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2382026-05-19T05:11:10.692Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.267637789Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"client":"0","":"downstairs","session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2392026-05-19T05:11:10.692Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.267647798Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"client":"1","":"downstairs","session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2402026-05-19T05:11:10.692Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.267658357Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"client":"2","":"downstairs","session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2412026-05-19T05:11:10.692Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.267672747Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2422026-05-19T05:11:10.692Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.267682526Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2432026-05-19T05:11:10.692Z{"msg":"7cb60a63-9787-4f59-8d59-b2f624b53263 is now active with session: 79fee139-4360-425d-aae5-649d98242bf9","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.267696965Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2442026-05-19T05:11:10.692Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.267710014Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"session_id":"79fee139-4360-425d-aae5-649d98242bf9"}
2452026-05-19T05:11:10.692Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.267719784Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182}
2462026-05-19T05:11:10.692Z{"msg":"Activated sub_volume 7cb60a63-9787-4f59-8d59-b2f624b53263","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.267732293Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182}
2472026-05-19T05:11:10.692Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.267740713Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"task":"crutest"}
2482026-05-19T05:11:10.692ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
2492026-05-19T05:11:10.692Z{"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.267914483Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"task":"crutest"}
2502026-05-19T05:11:10.711ZRead and Verify all blocks (0..100 range:false)
2512026-05-19T05:11:10.759Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30----------------------------------------------------------------
2522026-05-19T05:11:10.759Z,"time":"2026-05-19T05:11:09.340134978Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8 Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2532026-05-19T05:11:10.759Z"States:,"pid": Active1182 Active Active
2542026-05-19T05:11:10.759Z,"task":"JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2552026-05-19T05:11:10.759Zcrutest"}
2562026-05-19T05:11:10.759Z 1002 Acked Read 50{ Done" Donemsg" Done:" false
2572026-05-19T05:11:10.759ZCLIENT: Tests done. All submitted work has been ACK'd","v":0,"name":"crucible"," STATES DS:0 DS:1 DS:2 TOTAL
2582026-05-19T05:11:10.759Zlevel":30 Sent 0 0 0 0
2592026-05-19T05:11:10.759Z Done 1 1 1 3
2602026-05-19T05:11:10.759Z, Skipped " 0 time": 0 " 0 0
2612026-05-19T05:11:10.759Z2026-05-19T05:11:09.340167156Z" Error ," 0 0 hostname 0 " 0
2622026-05-19T05:11:10.759Z:"Last Flush: Some(JobId(1001)) w-01KRZA15QNXN04FPEYN6BYX5W8"Some(JobId(1001)) ,"Some(JobId(1001)) pid"
2632026-05-19T05:11:10.759Z:Downstairs last five completed:1182,"task":"crutest" 1001} 1000
2642026-05-19T05:11:10.759Z
2652026-05-19T05:11:10.759ZUpstairs last five completed: 1002 1001 1000
2662026-05-19T05:11:10.762Z----------------------------------------------------------------
2672026-05-19T05:11:10.762Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2682026-05-19T05:11:10.762ZStates: Active Active Active
2692026-05-19T05:11:10.762ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2702026-05-19T05:11:10.762Z 1002 Acked Read 50 Done Done Done false
2712026-05-19T05:11:10.762Z STATES DS:0 DS:1 DS:2 TOTAL
2722026-05-19T05:11:10.762Z Sent 0 0 0 0
2732026-05-19T05:11:10.763Z Done 1 1 1 3
2742026-05-19T05:11:10.763Z Skipped 0 0 0 0
2752026-05-19T05:11:10.763Z Error 0 0 0 0
2762026-05-19T05:11:10.763ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2772026-05-19T05:11:10.763ZDownstairs last five completed: 1001 1000
2782026-05-19T05:11:10.763ZUpstairs last five completed: 1002 1001 1000
2792026-05-19T05:11:10.763Z{"msg":"CLIENT: Up:0 ds:2 act:6","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:09.340265991Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"task":"crutest"}
2802026-05-19T05:11:14.760Z----------------------------------------------------------------
2812026-05-19T05:11:14.760Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2822026-05-19T05:11:14.761ZDownstairs last five completed: 1003 1002 1001 1000
2832026-05-19T05:11:14.761ZUpstairs last five completed: 1003 1002 1001 1000
2842026-05-19T05:11:14.761Z----------------------------------------------------------------
2852026-05-19T05:11:14.761Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2862026-05-19T05:11:14.761ZDownstairs last five completed: 1003 1002 1001 1000
2872026-05-19T05:11:14.761ZUpstairs last five completed: 1003 1002 1001 1000
2882026-05-19T05:11:14.764Z{"msg":"CLIENT: Up:0 ds:0 act:6","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.341416526Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"task":"crutest"}
2892026-05-19T05:11:14.764Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.341483732Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182,"task":"crutest"}
2902026-05-19T05:11:14.764Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-19T05:11:13.341503611Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1182}
2912026-05-19T05:11:14.769ZRepair setup passed
2922026-05-19T05:11:14.772ZCopy the region for /var/tmp/test_up-build/8830
2932026-05-19T05:11:14.775Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2942026-05-19T05:11:14.775Z/var/tmp/bins/crutest fill -g 32 -q --key RqBeYhUu4wg0w+PD4TB5syu41lzm6qyNsK0V6PdJSVE= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2952026-05-19T05:11:14.793Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.373975976Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"task":"crutest"}
2962026-05-19T05:11:14.796Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.374252141Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"task":"crutest"}
2972026-05-19T05:11:14.796Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.374976469Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"task":"crutest"}
2982026-05-19T05:11:14.796Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.375224205Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188}
2992026-05-19T05:11:14.796Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4196b24f875a2f584277e37760a6e226abbc3bf5\",\n git_commit_timestamp: \"2026-05-19T04:52:46.000000000Z\",\n git_branch: \"renovate/tracing-subscriber-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.375269052Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188}
3002026-05-19T05:11:14.796Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.375283022Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188}
3012026-05-19T05:11:14.796Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.375295621Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188}
3022026-05-19T05:11:14.796Z{"msg":"Crucible 9bfae2ec-49c1-449c-a32d-7e66fb8945e7 has session id: efe75954-f6fa-4cb3-b236-d656aa50cd90","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.375705088Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3032026-05-19T05:11:14.796Z{"msg":"Upstairs opts: Upstairs UUID: 9bfae2ec-49c1-449c-a32d-7e66fb8945e7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.375731726Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3042026-05-19T05:11:14.796Z{"msg":"Crucible stats registered with UUID: 9bfae2ec-49c1-449c-a32d-7e66fb8945e7","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.375741865Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3052026-05-19T05:11:14.796Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-19T05:11:13.375755545Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3062026-05-19T05:11:14.796Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.375904386Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3072026-05-19T05:11:14.796Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.375928295Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3082026-05-19T05:11:14.796Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.375938784Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3092026-05-19T05:11:14.796Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.3768916Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"task":"crutest"}
3102026-05-19T05:11:14.796Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.376932577Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188}
3112026-05-19T05:11:14.796Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4196b24f875a2f584277e37760a6e226abbc3bf5\",\n git_commit_timestamp: \"2026-05-19T04:52:46.000000000Z\",\n git_branch: \"renovate/tracing-subscriber-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.376958446Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188}
3122026-05-19T05:11:14.796Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.376971275Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188}
3132026-05-19T05:11:14.800Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.376983555Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188}
3142026-05-19T05:11:14.800Z{"msg":"Crucible 9bfae2ec-49c1-449c-a32d-7e66fb8945e7 has session id: 5af01baa-2355-41d5-b8c3-9536c363ffa5","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.377191903Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3152026-05-19T05:11:14.800Z{"msg":"Upstairs opts: Upstairs UUID: 9bfae2ec-49c1-449c-a32d-7e66fb8945e7, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.37723135Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3162026-05-19T05:11:14.800Z{"msg":"Crucible stats registered with UUID: 9bfae2ec-49c1-449c-a32d-7e66fb8945e7","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.377251339Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3172026-05-19T05:11:14.800Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-19T05:11:13.377261539Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3182026-05-19T05:11:14.800Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.377274868Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3192026-05-19T05:11:14.800Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.377285597Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3202026-05-19T05:11:14.800Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.377299936Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3212026-05-19T05:11:14.800Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.377396861Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"task":"crutest"}
3222026-05-19T05:11:14.800Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.377437049Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188}
3232026-05-19T05:11:14.800Z{"msg":"9bfae2ec-49c1-449c-a32d-7e66fb8945e7 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.377460187Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3242026-05-19T05:11:14.800Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.377470217Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"client":"0","":"downstairs","session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3252026-05-19T05:11:14.800Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.377485226Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"client":"1","":"downstairs","session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3262026-05-19T05:11:14.800Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.377495235Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"client":"2","":"downstairs","session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3272026-05-19T05:11:14.800Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.377505425Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3282026-05-19T05:11:14.800Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.377515424Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3292026-05-19T05:11:14.800Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.377529223Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3302026-05-19T05:11:14.800Z{"msg":"ds_connection connected from Ok(127.0.0.1:42628)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.37759094Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3312026-05-19T05:11:14.800Z{"msg":"ds_connection connected from Ok(127.0.0.1:54285)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.377630978Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3322026-05-19T05:11:14.801Z{"msg":"ds_connection connected from Ok(127.0.0.1:58417)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.377650066Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3332026-05-19T05:11:14.801Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.37828434Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"client":"2","":"downstairs","session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3342026-05-19T05:11:14.801Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.378319628Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"client":"0","":"downstairs","session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3352026-05-19T05:11:14.801Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.378339267Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"client":"1","":"downstairs","session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3362026-05-19T05:11:14.801Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.37845892Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3372026-05-19T05:11:14.801Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.378506558Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3382026-05-19T05:11:14.801Z{"msg":"[0]R flush_numbers: [4000, 4000, 4000, 4000, 4000]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.378536386Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"downstairs","":"downstairs","session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3392026-05-19T05:11:14.801Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.378552085Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"downstairs","":"downstairs","session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3402026-05-19T05:11:14.801Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.378566554Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"downstairs","":"downstairs","session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3412026-05-19T05:11:14.801Z{"msg":"[1]R flush_numbers: [4000, 4000, 4000, 4000, 4000]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.378576864Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"downstairs","":"downstairs","session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3422026-05-19T05:11:14.801Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.378590853Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"downstairs","":"downstairs","session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3432026-05-19T05:11:14.801Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.378600942Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"downstairs","":"downstairs","session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3442026-05-19T05:11:14.801Z{"msg":"[2]R flush_numbers: [4000, 4000, 4000, 4000, 4000]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.378611372Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"downstairs","":"downstairs","session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3452026-05-19T05:11:14.801Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.378621451Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"downstairs","":"downstairs","session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3462026-05-19T05:11:14.801Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.37863181Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"downstairs","":"downstairs","session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3472026-05-19T05:11:14.801Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.37864567Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"downstairs","":"downstairs","session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3482026-05-19T05:11:14.801Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.378655169Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"downstairs","":"downstairs","session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3492026-05-19T05:11:14.801Z{"msg":"Next flush: 4001","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.378664939Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"downstairs","":"downstairs","session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3502026-05-19T05:11:14.801Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.378677948Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"downstairs","":"downstairs","session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3512026-05-19T05:11:14.801Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.378687797Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3522026-05-19T05:11:14.801Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.378740374Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"client":"0","":"downstairs","session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3532026-05-19T05:11:14.801Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.378763763Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"client":"1","":"downstairs","session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3542026-05-19T05:11:14.801Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.378774592Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"client":"2","":"downstairs","session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3552026-05-19T05:11:14.801Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.378784962Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3562026-05-19T05:11:14.801Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.378794181Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3572026-05-19T05:11:14.801Z{"msg":"9bfae2ec-49c1-449c-a32d-7e66fb8945e7 is now active with session: efe75954-f6fa-4cb3-b236-d656aa50cd90","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.37880886Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3582026-05-19T05:11:14.801Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.37881805Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"session_id":"efe75954-f6fa-4cb3-b236-d656aa50cd90"}
3592026-05-19T05:11:14.801Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.378878116Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188}
3602026-05-19T05:11:14.801Z{"msg":"Activated sub_volume 9bfae2ec-49c1-449c-a32d-7e66fb8945e7","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.378899405Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188}
3612026-05-19T05:11:14.801Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.378913204Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188}
3622026-05-19T05:11:14.801Z{"msg":"9bfae2ec-49c1-449c-a32d-7e66fb8945e7 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.378921764Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3632026-05-19T05:11:14.801Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.378931443Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"client":"0","":"downstairs","session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3642026-05-19T05:11:14.801Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.378942053Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"client":"1","":"downstairs","session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3652026-05-19T05:11:14.801Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.378956212Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"client":"2","":"downstairs","session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3662026-05-19T05:11:14.801Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.378973291Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3672026-05-19T05:11:14.801Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.37898347Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3682026-05-19T05:11:14.801Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.378998Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3692026-05-19T05:11:14.802Z{"msg":"ds_connection connected from Ok(127.0.0.1:36709)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.379008629Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3702026-05-19T05:11:14.802Z{"msg":"ds_connection connected from Ok(127.0.0.1:61658)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.379022988Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3712026-05-19T05:11:14.802Z{"msg":"ds_connection connected from Ok(127.0.0.1:41505)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.379037527Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3722026-05-19T05:11:14.802Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.379236456Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"client":"1","":"downstairs","session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3732026-05-19T05:11:14.802Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.379271244Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"client":"0","":"downstairs","session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3742026-05-19T05:11:14.802Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.379359099Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"client":"2","":"downstairs","session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3752026-05-19T05:11:14.802Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.379414256Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3762026-05-19T05:11:14.802Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.379438734Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3772026-05-19T05:11:14.802Z{"msg":"[0]R flush_numbers: [5323, 5323, 5323, 5323, 5323]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.379524649Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"downstairs","":"downstairs","session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3782026-05-19T05:11:14.802Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.379540239Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"downstairs","":"downstairs","session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3792026-05-19T05:11:14.802Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.379550848Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"downstairs","":"downstairs","session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3802026-05-19T05:11:14.802Z{"msg":"[1]R flush_numbers: [5323, 5323, 5323, 5323, 5323]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.379565207Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"downstairs","":"downstairs","session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3812026-05-19T05:11:14.802Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.379575267Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"downstairs","":"downstairs","session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3822026-05-19T05:11:14.802Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.379593605Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"downstairs","":"downstairs","session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3832026-05-19T05:11:14.802Z{"msg":"[2]R flush_numbers: [5323, 5323, 5323, 5323, 5323]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.379608295Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"downstairs","":"downstairs","session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3842026-05-19T05:11:14.802Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.379618204Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"downstairs","":"downstairs","session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3852026-05-19T05:11:14.802Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.379632723Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"downstairs","":"downstairs","session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3862026-05-19T05:11:14.802Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.379642153Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"downstairs","":"downstairs","session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3872026-05-19T05:11:14.802Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.379652272Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"downstairs","":"downstairs","session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3882026-05-19T05:11:14.802Z{"msg":"Next flush: 5324","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.379662052Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"downstairs","":"downstairs","session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3892026-05-19T05:11:14.802Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.379676151Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"":"downstairs","":"downstairs","session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3902026-05-19T05:11:14.802Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.37968629Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3912026-05-19T05:11:14.802Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.379699869Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"client":"0","":"downstairs","session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3922026-05-19T05:11:14.802Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.379709699Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"client":"1","":"downstairs","session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3932026-05-19T05:11:14.802Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.379723798Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"client":"2","":"downstairs","session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3942026-05-19T05:11:14.802Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.379733877Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3952026-05-19T05:11:14.802Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.379747877Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3962026-05-19T05:11:14.802Z{"msg":"9bfae2ec-49c1-449c-a32d-7e66fb8945e7 is now active with session: 5af01baa-2355-41d5-b8c3-9536c363ffa5","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.379758366Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3972026-05-19T05:11:14.802Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.379771555Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"session_id":"5af01baa-2355-41d5-b8c3-9536c363ffa5"}
3982026-05-19T05:11:14.802Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.379780895Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188}
3992026-05-19T05:11:14.803Z{"msg":"Activated sub_volume 9bfae2ec-49c1-449c-a32d-7e66fb8945e7","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.379793474Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188}
4002026-05-19T05:11:14.803Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.379801974Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"task":"crutest"}
4012026-05-19T05:11:14.803ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
4022026-05-19T05:11:14.803ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4032026-05-19T05:11:14.803Z{"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.380303005Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"task":"crutest"}
4042026-05-19T05:11:14.824ZRead and Verify all blocks (0..100 range:false)
4052026-05-19T05:11:14.871Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30----------------------------------------------------------------
4062026-05-19T05:11:14.871Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4072026-05-19T05:11:14.871ZStates: Active Active Active
4082026-05-19T05:11:14.871ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4092026-05-19T05:11:14.871Z,"time":" 1002 Acked Read 502026-05-19T05:11:13.451892346Z" Done Done, Done" false
4102026-05-19T05:11:14.871Zhostname":"w-01KRZA15QNXN04FPEYN6BYX5W8"," STATES DS:0 DS:1 DS:2 TOTAL
4112026-05-19T05:11:14.871Zpid" Sent : 0 1188 0 0 0
4122026-05-19T05:11:14.871Z," Done 1 task 1 ":" 1 3
4132026-05-19T05:11:14.871Zcrutest Skipped " 0 0 0 0
4142026-05-19T05:11:14.871Z} Error
4152026-05-19T05:11:14.871Z 0 0 0 0
4162026-05-19T05:11:14.874Z{Last Flush: "Some(JobId(1001)) msgSome(JobId(1001)) "Some(JobId(1001)) :"
4172026-05-19T05:11:14.874ZCLIENT: Tests done. All submitted work has been ACK'd"Downstairs last five completed:,"v":0,"name":"crucible" 1001, 1000"
4182026-05-19T05:11:14.874Zlevel":30Upstairs last five completed: 1002 1001 1000
4192026-05-19T05:11:14.874Z,"time":"2026-05-19T05:11:13.451931054Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"task":"crutest"}
4202026-05-19T05:11:14.874Z----------------------------------------------------------------
4212026-05-19T05:11:14.874Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4222026-05-19T05:11:14.874ZStates: Active Active Active
4232026-05-19T05:11:14.874ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4242026-05-19T05:11:14.874Z 1002 Acked Read 50 Done Done Done false
4252026-05-19T05:11:14.874Z STATES DS:0 DS:1 DS:2 TOTAL
4262026-05-19T05:11:14.874Z Sent 0 0 0 0
4272026-05-19T05:11:14.874Z Done 1 1 1 3
4282026-05-19T05:11:14.874Z Skipped 0 0 0 0
4292026-05-19T05:11:14.874Z Error 0 0 0 0
4302026-05-19T05:11:14.874ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4312026-05-19T05:11:14.875ZDownstairs last five completed: 1001 1000
4322026-05-19T05:11:14.875ZUpstairs last five completed: 1002 1001 1000
4332026-05-19T05:11:14.875Z{"msg":"CLIENT: Up:0 ds:2 act:6","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:13.452049377Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"task":"crutest"}
4342026-05-19T05:11:18.873Z----------------------------------------------------------------
4352026-05-19T05:11:18.874Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4362026-05-19T05:11:18.874ZDownstairs last five completed: 1003 1002 1001 1000
4372026-05-19T05:11:18.874ZUpstairs last five completed: 1003 1002 1001 1000
4382026-05-19T05:11:18.874Z----------------------------------------------------------------
4392026-05-19T05:11:18.874Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4402026-05-19T05:11:18.874ZDownstairs last five completed: 1003 1002 1001 1000
4412026-05-19T05:11:18.874ZUpstairs last five completed: 1003 1002 1001 1000
4422026-05-19T05:11:18.877Z{"msg":"CLIENT: Up:0 ds:0 act:6","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.454100971Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"task":"crutest"}
4432026-05-19T05:11:18.877Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.454173787Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188,"task":"crutest"}
4442026-05-19T05:11:18.877Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-19T05:11:17.454197096Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188}
4452026-05-19T05:11:18.877Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-19T05:11:17.454316979Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1188}
4462026-05-19T05:11:18.879ZRepair part 1 passed
4472026-05-19T05:11:18.879Z
4482026-05-19T05:11:18.879ZKill the current downstairs
4492026-05-19T05:11:18.922ZDownstairs 2 stopped
4502026-05-19T05:11:18.923Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
4512026-05-19T05:11:18.925ZNow put back the original so we have a mismatch
4522026-05-19T05:11:18.925Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
4532026-05-19T05:11:18.925ZRestart downstairs with old directory
4542026-05-19T05:11:18.938Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4552026-05-19T05:11:18.950ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4562026-05-19T05:11:18.953Z 0 000-009 32 32 31 4001 4001 4000 F F F <---
4572026-05-19T05:11:18.953Z 1 010-019 32 32 31 4001 4001 4000 F F F <---
4582026-05-19T05:11:18.953Z 2 020-029 32 32 31 4001 4001 4000 F F F <---
4592026-05-19T05:11:18.953Z 3 030-039 32 32 31 4001 4001 4000 F F F <---
4602026-05-19T05:11:18.953Z 4 040-049 32 32 31 4001 4001 4000 F F F <---
4612026-05-19T05:11:18.953ZMax gen: 32, Max flush: 4001
4622026-05-19T05:11:18.953ZError: Difference in extent metadata found!
4632026-05-19T05:11:18.953Zdump test found error as expected
4642026-05-19T05:11:18.953Z
4652026-05-19T05:11:18.953Z
4662026-05-19T05:11:18.953Z/var/tmp/bins/crutest verify --range -g 33 -q --key RqBeYhUu4wg0w+PD4TB5syu41lzm6qyNsK0V6PdJSVE= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
4672026-05-19T05:11:18.971Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.551927964Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"task":"crutest"}
4682026-05-19T05:11:18.974Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.552238766Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"task":"crutest"}
4692026-05-19T05:11:18.974Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.552928427Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"task":"crutest"}
4702026-05-19T05:11:18.974Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.553180533Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196}
4712026-05-19T05:11:18.974Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4196b24f875a2f584277e37760a6e226abbc3bf5\",\n git_commit_timestamp: \"2026-05-19T04:52:46.000000000Z\",\n git_branch: \"renovate/tracing-subscriber-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.55322954Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196}
4722026-05-19T05:11:18.974Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.553239629Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196}
4732026-05-19T05:11:18.974Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.553252099Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196}
4742026-05-19T05:11:18.974Z{"msg":"Crucible 911cb948-6675-4f5e-9976-ae3fa5ea2c62 has session id: d23df1d7-1b67-45b2-92db-4d7343312633","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.553654716Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
4752026-05-19T05:11:18.974Z{"msg":"Upstairs opts: Upstairs UUID: 911cb948-6675-4f5e-9976-ae3fa5ea2c62, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.553674264Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
4762026-05-19T05:11:18.974Z{"msg":"Crucible stats registered with UUID: 911cb948-6675-4f5e-9976-ae3fa5ea2c62","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.553684684Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
4772026-05-19T05:11:18.974Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-19T05:11:17.553694153Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
4782026-05-19T05:11:18.974Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.553828656Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
4792026-05-19T05:11:18.974Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.553845435Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
4802026-05-19T05:11:18.974Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.553855894Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
4812026-05-19T05:11:18.975Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.554847327Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"task":"crutest"}
4822026-05-19T05:11:18.975Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.554910194Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196}
4832026-05-19T05:11:18.975Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4196b24f875a2f584277e37760a6e226abbc3bf5\",\n git_commit_timestamp: \"2026-05-19T04:52:46.000000000Z\",\n git_branch: \"renovate/tracing-subscriber-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.554933722Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196}
4842026-05-19T05:11:18.975Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.554946172Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196}
4852026-05-19T05:11:18.975Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.554957811Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196}
4862026-05-19T05:11:18.978Z{"msg":"Crucible 911cb948-6675-4f5e-9976-ae3fa5ea2c62 has session id: 815f1dad-6453-4b17-9b95-9c239c6d9532","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.555244415Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
4872026-05-19T05:11:18.978Z{"msg":"Upstairs opts: Upstairs UUID: 911cb948-6675-4f5e-9976-ae3fa5ea2c62, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.555278393Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
4882026-05-19T05:11:18.978Z{"msg":"Crucible stats registered with UUID: 911cb948-6675-4f5e-9976-ae3fa5ea2c62","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.555302721Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
4892026-05-19T05:11:18.978Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-19T05:11:17.555316111Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
4902026-05-19T05:11:18.978Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.55532489Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
4912026-05-19T05:11:18.978Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.555336249Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
4922026-05-19T05:11:18.978Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.555347049Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
4932026-05-19T05:11:18.978Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.555468982Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"task":"crutest"}
4942026-05-19T05:11:18.978Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.555482391Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196}
4952026-05-19T05:11:18.978Z{"msg":"911cb948-6675-4f5e-9976-ae3fa5ea2c62 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.55549875Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
4962026-05-19T05:11:18.978Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.55550848Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"client":"0","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
4972026-05-19T05:11:18.978Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.555518059Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"client":"1","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
4982026-05-19T05:11:18.978Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.555527969Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"client":"2","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
4992026-05-19T05:11:18.978Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.555537358Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5002026-05-19T05:11:18.979Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.555547677Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5012026-05-19T05:11:18.979Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.555561757Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5022026-05-19T05:11:18.979Z{"msg":"ds_connection connected from Ok(127.0.0.1:37513)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.555623343Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5032026-05-19T05:11:18.979Z{"msg":"ds_connection connected from Ok(127.0.0.1:59710)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.555641942Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5042026-05-19T05:11:18.979Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-05-19T05:11:17.555654281Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5052026-05-19T05:11:18.979Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-05-19T05:11:17.555666891Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5062026-05-19T05:11:18.979Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-05-19T05:11:17.5556793Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5072026-05-19T05:11:18.979Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.555751336Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5082026-05-19T05:11:18.979Z{"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.555810122Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5092026-05-19T05:11:18.979Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.556217799Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"client":"1","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5102026-05-19T05:11:18.979Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.556240418Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"client":"0","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5112026-05-19T05:11:18.979Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.55637355Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5122026-05-19T05:11:18.979Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:17.556391459Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5132026-05-19T05:11:28.977Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.557095097Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5142026-05-19T05:11:28.977Z{"msg":"ds_connection connected from Ok(127.0.0.1:51665)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.557191822Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5152026-05-19T05:11:28.982Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.558631399Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"client":"2","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5162026-05-19T05:11:28.982Z{"msg":"[0]R flush_numbers: [4001, 4001, 4001, 4001, 4001]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.558927042Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"downstairs","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5172026-05-19T05:11:28.982Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.558947711Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"downstairs","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5182026-05-19T05:11:28.982Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.55896549Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"downstairs","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5192026-05-19T05:11:28.982Z{"msg":"[1]R flush_numbers: [4001, 4001, 4001, 4001, 4001]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.558985089Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"downstairs","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5202026-05-19T05:11:28.982Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.559004278Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"downstairs","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5212026-05-19T05:11:28.982Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.559022247Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"downstairs","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5222026-05-19T05:11:28.982Z{"msg":"[2]R flush_numbers: [4000, 4000, 4000, 4000, 4000]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.559041276Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"downstairs","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5232026-05-19T05:11:28.982Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.559059625Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"downstairs","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5242026-05-19T05:11:28.982Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.559078094Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"downstairs","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5252026-05-19T05:11:28.982Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.559095313Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"downstairs","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5262026-05-19T05:11:28.982Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.559114042Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"downstairs","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5272026-05-19T05:11:28.982Z{"msg":"Next flush: 4002","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.559131911Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"downstairs","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5282026-05-19T05:11:28.983Z{"msg":"extent 0 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4000, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.559165579Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5292026-05-19T05:11:28.983Z{"msg":"extent:0 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4000, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.559199967Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5302026-05-19T05:11:28.983Z{"msg":"found dest for source 0 for extent at index 0 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.559226395Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5312026-05-19T05:11:28.983Z{"msg":"extent 1 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4000, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.559266563Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5322026-05-19T05:11:28.983Z{"msg":"extent:1 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4000, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.559298991Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5332026-05-19T05:11:28.983Z{"msg":"found dest for source 0 for extent at index 1 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.55932611Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5342026-05-19T05:11:28.983Z{"msg":"extent 2 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4000, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.559358728Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5352026-05-19T05:11:28.983Z{"msg":"extent:2 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4000, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.559391136Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5362026-05-19T05:11:28.983Z{"msg":"found dest for source 0 for extent at index 2 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.559410195Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5372026-05-19T05:11:28.983Z{"msg":"extent 3 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4000, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.559438003Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5382026-05-19T05:11:28.983Z{"msg":"extent:3 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4000, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.559462672Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5392026-05-19T05:11:28.983Z{"msg":"found dest for source 0 for extent at index 3 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.55948691Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5402026-05-19T05:11:28.983Z{"msg":"extent 4 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4000, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.559517279Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5412026-05-19T05:11:28.983Z{"msg":"extent:4 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4000, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.559547627Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5422026-05-19T05:11:28.983Z{"msg":"found dest for source 0 for extent at index 4 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.559571796Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5432026-05-19T05:11:28.983Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.559604804Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"client":"0","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5442026-05-19T05:11:28.983Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.559629682Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"client":"1","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5452026-05-19T05:11:28.983Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.559653741Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"client":"2","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5462026-05-19T05:11:28.983Z{"msg":"Full repair list: {ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.559695958Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"downstairs","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5472026-05-19T05:11:28.983Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 4002, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 4002, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 4002, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 4002, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 4002, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.559857929Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"downstairs","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5482026-05-19T05:11:28.983Z{"msg":"starting reconciliation 58712b09-00e1-40e8-a4c7-8bc7415e742b: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.559877408Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"downstairs","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5492026-05-19T05:11:28.983Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.559894817Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"client":"2","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5502026-05-19T05:11:28.988Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.568446449Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"client":"2","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5512026-05-19T05:11:28.991Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.571014642Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"client":"2","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5522026-05-19T05:11:28.994Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.573099923Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"client":"2","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5532026-05-19T05:11:28.996Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.575121438Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"client":"2","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5542026-05-19T05:11:29.000Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.576847829Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"downstairs","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5552026-05-19T05:11:29.000Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.576865748Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"client":"0","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5562026-05-19T05:11:29.001Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.576877157Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"client":"1","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5572026-05-19T05:11:29.001Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.576893176Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"client":"2","":"downstairs","session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5582026-05-19T05:11:29.001Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.576904186Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5592026-05-19T05:11:29.001Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.576919455Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5602026-05-19T05:11:29.001Z{"msg":"911cb948-6675-4f5e-9976-ae3fa5ea2c62 is now active with session: d23df1d7-1b67-45b2-92db-4d7343312633","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.576931624Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"session_id":"d23df1d7-1b67-45b2-92db-4d7343312633"}
5612026-05-19T05:11:29.001Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.576946993Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196}
5622026-05-19T05:11:29.001Z{"msg":"Activated sub_volume 911cb948-6675-4f5e-9976-ae3fa5ea2c62","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.576956583Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196}
5632026-05-19T05:11:29.001Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.576969852Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196}
5642026-05-19T05:11:29.001Z{"msg":"911cb948-6675-4f5e-9976-ae3fa5ea2c62 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.576979171Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5652026-05-19T05:11:29.001Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.576994231Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"client":"0","":"downstairs","session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5662026-05-19T05:11:29.001Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.57700536Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"client":"1","":"downstairs","session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5672026-05-19T05:11:29.001Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.577016769Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"client":"2","":"downstairs","session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5682026-05-19T05:11:29.001Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.577027729Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5692026-05-19T05:11:29.001Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.577040068Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5702026-05-19T05:11:29.001Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.577056037Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5712026-05-19T05:11:29.001Z{"msg":"ds_connection connected from Ok(127.0.0.1:58459)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.577067896Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5722026-05-19T05:11:29.001Z{"msg":"ds_connection connected from Ok(127.0.0.1:41407)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.577079976Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5732026-05-19T05:11:29.001Z{"msg":"ds_connection connected from Ok(127.0.0.1:56147)","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.577091485Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5742026-05-19T05:11:29.001Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.577487522Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"client":"1","":"downstairs","session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5752026-05-19T05:11:29.001Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.577517071Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"client":"0","":"downstairs","session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5762026-05-19T05:11:29.001Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.577598016Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"client":"2","":"downstairs","session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5772026-05-19T05:11:29.001Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.577663762Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5782026-05-19T05:11:29.002Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.577686461Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5792026-05-19T05:11:29.002Z{"msg":"[0]R flush_numbers: [5324, 5324, 5324, 5324, 5324]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.577783565Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"downstairs","":"downstairs","session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5802026-05-19T05:11:29.002Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.577794655Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"downstairs","":"downstairs","session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5812026-05-19T05:11:29.002Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.577805134Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"downstairs","":"downstairs","session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5822026-05-19T05:11:29.002Z{"msg":"[1]R flush_numbers: [5324, 5324, 5324, 5324, 5324]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.577815704Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"downstairs","":"downstairs","session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5832026-05-19T05:11:29.002Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.577825843Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"downstairs","":"downstairs","session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5842026-05-19T05:11:29.002Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.577835682Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"downstairs","":"downstairs","session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5852026-05-19T05:11:29.002Z{"msg":"[2]R flush_numbers: [5324, 5324, 5324, 5324, 5324]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.577846242Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"downstairs","":"downstairs","session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5862026-05-19T05:11:29.002Z{"msg":"[2]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.577856331Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"downstairs","":"downstairs","session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5872026-05-19T05:11:29.002Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.577866801Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"downstairs","":"downstairs","session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5882026-05-19T05:11:29.002Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.57787644Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"downstairs","":"downstairs","session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5892026-05-19T05:11:29.002Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.57788691Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"downstairs","":"downstairs","session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5902026-05-19T05:11:29.002Z{"msg":"Next flush: 5325","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.577900339Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"downstairs","":"downstairs","session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5912026-05-19T05:11:29.002Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.577909718Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"":"downstairs","":"downstairs","session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5922026-05-19T05:11:29.002Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.577923577Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5932026-05-19T05:11:29.002Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.577932697Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"client":"0","":"downstairs","session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5942026-05-19T05:11:29.002Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.577942976Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"client":"1","":"downstairs","session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5952026-05-19T05:11:29.002Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.577952836Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"client":"2","":"downstairs","session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5962026-05-19T05:11:29.002Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.577963145Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5972026-05-19T05:11:29.002Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.577972755Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5982026-05-19T05:11:29.002Z{"msg":"911cb948-6675-4f5e-9976-ae3fa5ea2c62 is now active with session: 815f1dad-6453-4b17-9b95-9c239c6d9532","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.577983804Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
5992026-05-19T05:11:29.002Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.577992924Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"session_id":"815f1dad-6453-4b17-9b95-9c239c6d9532"}
6002026-05-19T05:11:29.002Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.578002763Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196}
6012026-05-19T05:11:29.002Z{"msg":"Activated sub_volume 911cb948-6675-4f5e-9976-ae3fa5ea2c62","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.578011052Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196}
6022026-05-19T05:11:29.002Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.578019382Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"task":"crutest"}
6032026-05-19T05:11:29.002ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
6042026-05-19T05:11:29.002ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
6052026-05-19T05:11:29.002ZRead and Verify all blocks (0..100 range:true)
6062026-05-19T05:11:29.045Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.625488781Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"task":"crutest"}
6072026-05-19T05:11:29.045Z{"msg":"Verify test completed","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.62551094Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"task":"crutest"}
6082026-05-19T05:11:29.045Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.625531089Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"task":"crutest"}
6092026-05-19T05:11:29.048Z{"msg":"CLIENT: Tests done. All submitted work has been ACK'd","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.625540608Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"task":"crutest"}
6102026-05-19T05:11:29.049Z----------------------------------------------------------------
6112026-05-19T05:11:29.049Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6122026-05-19T05:11:29.049ZStates: Active Active Active
6132026-05-19T05:11:29.049ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6142026-05-19T05:11:29.049Z 1000 Acked Read 50 Done Done Done false
6152026-05-19T05:11:29.049Z STATES DS:0 DS:1 DS:2 TOTAL
6162026-05-19T05:11:29.049Z Sent 0 0 0 0
6172026-05-19T05:11:29.049Z Done 1 1 1 3
6182026-05-19T05:11:29.049Z Skipped 0 0 0 0
6192026-05-19T05:11:29.049Z Error 0 0 0 0
6202026-05-19T05:11:29.049ZLast Flush: None None None
6212026-05-19T05:11:29.049ZDownstairs last five completed:
6222026-05-19T05:11:29.049ZUpstairs last five completed: 1000
6232026-05-19T05:11:29.049Z----------------------------------------------------------------
6242026-05-19T05:11:29.049Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6252026-05-19T05:11:29.049ZStates: Active Active Active
6262026-05-19T05:11:29.049ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6272026-05-19T05:11:29.049Z 1000 Acked Read 50 Done Done Done false
6282026-05-19T05:11:29.049Z STATES DS:0 DS:1 DS:2 TOTAL
6292026-05-19T05:11:29.049Z Sent 0 0 0 0
6302026-05-19T05:11:29.049Z Done 1 1 1 3
6312026-05-19T05:11:29.049Z Skipped 0 0 0 0
6322026-05-19T05:11:29.049Z Error 0 0 0 0
6332026-05-19T05:11:29.049ZLast Flush: None None None
6342026-05-19T05:11:29.049ZDownstairs last five completed:
6352026-05-19T05:11:29.049ZUpstairs last five completed: 1000
6362026-05-19T05:11:29.049Z{"msg":"CLIENT: Up:0 ds:2 act:6","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:27.625691309Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"task":"crutest"}
6372026-05-19T05:11:33.047Z----------------------------------------------------------------
6382026-05-19T05:11:33.047Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6392026-05-19T05:11:33.047ZDownstairs last five completed: 1001 1000
6402026-05-19T05:11:33.047ZUpstairs last five completed: 1001 1000
6412026-05-19T05:11:33.047Z----------------------------------------------------------------
6422026-05-19T05:11:33.047Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6432026-05-19T05:11:33.047ZDownstairs last five completed: 1001 1000
6442026-05-19T05:11:33.047ZUpstairs last five completed: 1001 1000
6452026-05-19T05:11:33.050Z{"msg":"CLIENT: Up:0 ds:0 act:6","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:31.626759549Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"task":"crutest"}
6462026-05-19T05:11:33.050Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-05-19T05:11:31.626821966Z","hostname":"w-01KRZA15QNXN04FPEYN6BYX5W8","pid":1196,"task":"crutest"}
6472026-05-19T05:11:33.053ZRepair part 2 passed
6482026-05-19T05:11:33.053Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
6492026-05-19T05:11:33.066ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6502026-05-19T05:11:33.066Z 0 000-009 32 32 32 4001 4001 4001 F F F
6512026-05-19T05:11:33.066Z 1 010-019 32 32 32 4001 4001 4001 F F F
6522026-05-19T05:11:33.066Z 2 020-029 32 32 32 4001 4001 4001 F F F
6532026-05-19T05:11:33.066Z 3 030-039 32 32 32 4001 4001 4001 F F F
6542026-05-19T05:11:33.066Z 4 040-049 32 32 32 4001 4001 4001 F F F
6552026-05-19T05:11:33.069ZMax gen: 32, Max flush: 4001
6562026-05-19T05:11:33.069Zdump test passed
6572026-05-19T05:11:33.069Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
6582026-05-19T05:11:33.080Z Extent 1
6592026-05-19T05:11:33.083ZGEN 32 32 32
6602026-05-19T05:11:33.083ZFLUSH_ID 4001 4001 4001
6612026-05-19T05:11:33.083ZDIRTY
6622026-05-19T05:11:33.083Z
6632026-05-19T05:11:33.083ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
6642026-05-19T05:11:33.083Z010 A A A A A A
6652026-05-19T05:11:33.083Z011 A A A A A A
6662026-05-19T05:11:33.083Z012 A A A A A A
6672026-05-19T05:11:33.086Z013 A A A A A A
6682026-05-19T05:11:33.086Z014 A A A A A A
6692026-05-19T05:11:33.086Z015 A A A A A A
6702026-05-19T05:11:33.086Z016 A A A A A A
6712026-05-19T05:11:33.086Z017 A A A A A A
6722026-05-19T05:11:33.089Z018 A A A A A A
6732026-05-19T05:11:33.089Z019 A A A A A A
6742026-05-19T05:11:33.089Zdump extent test passed
6752026-05-19T05:11:33.092Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
6762026-05-19T05:11:33.099ZExtent 2 Block in extent 0 Actual block 20
6772026-05-19T05:11:33.099Z
6782026-05-19T05:11:33.102Z DATA SHA256 VER
6792026-05-19T05:11:33.102Z------ ---------------------------------------------------------------- ---
6802026-05-19T05:11:33.102Z 0 0ca040f8fe8649b079b2b66c06c6c3d1d3d70749999335383c3603a4a35e23c3 A
6812026-05-19T05:11:33.102Z 1 0ca040f8fe8649b079b2b66c06c6c3d1d3d70749999335383c3603a4a35e23c3 A
6822026-05-19T05:11:33.102Z 2 0ca040f8fe8649b079b2b66c06c6c3d1d3d70749999335383c3603a4a35e23c3 A
6832026-05-19T05:11:33.102Z
6842026-05-19T05:11:33.102ZNONCES 0 1 2 DIFF
6852026-05-19T05:11:33.102Z------ ------------------------ ------------------------ ------------------------ -----
6862026-05-19T05:11:33.102Z 0 d00556764930a496dbc56566 d00556764930a496dbc56566 d00556764930a496dbc56566
6872026-05-19T05:11:33.102Z
6882026-05-19T05:11:33.102Z TAGS 0 1 2 DIFF
6892026-05-19T05:11:33.102Z------ -------------------------------- -------------------------------- -------------------------------- -----
6902026-05-19T05:11:33.102Z 0 d68da4b387d7f2e5a3d07cadffb27259 d68da4b387d7f2e5a3d07cadffb27259 d68da4b387d7f2e5a3d07cadffb27259
6912026-05-19T05:11:33.102Z
6922026-05-19T05:11:33.102ZHASHES 0 1 2 DIFF
6932026-05-19T05:11:33.102Z------ ---------------- ---------------- ---------------- -----
6942026-05-19T05:11:33.102Z
6952026-05-19T05:11:33.105Zdump block test passed
6962026-05-19T05:11:33.105ZInitial upstairs tests have completed, stopping all downstairs
6972026-05-19T05:11:38.124ZCreating 7 larger downstairs regions
6982026-05-19T05:11:38.326ZStarting 7 downstairs
6992026-05-19T05:11:43.332Zdsc restarted at PID: 1212
7002026-05-19T05:11:43.429ZNow do the replace-reconcile test
7012026-05-19T05:11:43.442ZUsing 8870 for the replacement port
7022026-05-19T05:15:11.851ZCompleted test: replace-reconcile
7032026-05-19T05:15:11.854ZNow do the replace-before-active test
7042026-05-19T05:19:05.004ZCompleted test: replace-before-active
7052026-05-19T05:19:05.004ZAll tests have completed, stopping all downstairs
7062026-05-19T05:19:05.023Z
7072026-05-19T05:19:05.023ZAll Tests have passed
7082026-05-19T05:19:05.023Z11:34 Test duration
7092026-05-19T05:19:05.027Z
7102026-05-19T05:19:05.027Zreal 11:33.796143006
7112026-05-19T05:19:05.027Zuser 16:38.551405631
7122026-05-19T05:19:05.027Zsys 11.826209531
7132026-05-19T05:19:05.027Ztrap 0.414501549
7142026-05-19T05:19:05.029Ztflt 0.060460060
7152026-05-19T05:19:05.029Zdflt 0.018660699
7162026-05-19T05:19:05.029Zkflt 0.001369617
7172026-05-19T05:19:05.029Zlock 16:07:46.139578466
7182026-05-19T05:19:05.029Zslp 50:33.860181862
7192026-05-19T05:19:05.029Zlat 1:50.191052920
7202026-05-19T05:19:05.029Zstop 0.052799992
7212026-05-19T05:19:05.029Z+ echo 'test-up-2region-encrypted ends'
7222026-05-19T05:19:05.029Ztest-up-2region-encrypted ends
7232026-05-19T05:19:10.029Zprocess exited: duration 702875 ms, exit code 0
7242026-05-19T05:19:10.029Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
7252026-05-19T05:20:10.032Zexec warning: : stdout descriptor may be held open by a background process; giving up!
7262026-05-19T05:20:10.032Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
7272026-05-19T05:20:10.044Zfound 17 output files
7282026-05-19T05:20:10.044Zuploading: /tmp/test_up-build/dsc-out.txt (58377 bytes)
7292026-05-19T05:20:11.056Zuploaded: /tmp/test_up-build/dsc-out.txt
7302026-05-19T05:20:11.056Zuploading: /tmp/test_up-build/test_up_out.txt (23151576 bytes)
7312026-05-19T05:20:12.190Zuploaded: /tmp/test_up-build/test_up_out.txt
7322026-05-19T05:20:12.194Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (550952 bytes)
7332026-05-19T05:20:13.208Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
7342026-05-19T05:20:13.208Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (548341 bytes)
7352026-05-19T05:20:14.225Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
7362026-05-19T05:20:14.225Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (978684 bytes)
7372026-05-19T05:20:15.243Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
7382026-05-19T05:20:15.243Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (543069 bytes)
7392026-05-19T05:20:16.261Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
7402026-05-19T05:20:16.261Zuploading: /tmp/test_up-build/dsc/downstairs-8850.txt (2508 bytes)
7412026-05-19T05:20:17.269Zuploaded: /tmp/test_up-build/dsc/downstairs-8850.txt
7422026-05-19T05:20:17.269Zuploading: /tmp/test_up-build/dsc/downstairs-8860.txt (47334 bytes)
7432026-05-19T05:20:18.278Zuploaded: /tmp/test_up-build/dsc/downstairs-8860.txt
7442026-05-19T05:20:18.278Zuploading: /tmp/test_up-build/dsc/downstairs-8870.txt (3598171 bytes)
7452026-05-19T05:20:19.318Zuploaded: /tmp/test_up-build/dsc/downstairs-8870.txt
7462026-05-19T05:20:19.318Zuploading: /tmp/debug/df.txt (1270 bytes)
7472026-05-19T05:20:20.328Zuploaded: /tmp/debug/df.txt
7482026-05-19T05:20:20.328Zuploading: /tmp/debug/dtrace.txt (1044719 bytes)
7492026-05-19T05:20:21.348Zuploaded: /tmp/debug/dtrace.txt
7502026-05-19T05:20:21.348Zuploading: /tmp/debug/iostat.txt (224925 bytes)
7512026-05-19T05:20:21.356Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 224925 -> 228477
7522026-05-19T05:20:22.363Zuploaded: /tmp/debug/iostat.txt
7532026-05-19T05:20:22.363Zuploading: /tmp/debug/mpstat.txt (550445 bytes)
7542026-05-19T05:20:22.374Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 550445 -> 559870
7552026-05-19T05:20:23.382Zuploaded: /tmp/debug/mpstat.txt
7562026-05-19T05:20:23.382Zuploading: /tmp/debug/paging.txt (96257 bytes)
7572026-05-19T05:20:23.388Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 96257 -> 98127
7582026-05-19T05:20:24.394Zuploaded: /tmp/debug/paging.txt
7592026-05-19T05:20:24.394Zuploading: /tmp/debug/prstat.txt (1057345 bytes)
7602026-05-19T05:20:24.411Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 1057345 -> 1075404
7612026-05-19T05:20:25.417Zuploaded: /tmp/debug/prstat.txt
7622026-05-19T05:20:25.417Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
7632026-05-19T05:20:26.425Zuploaded: /tmp/debug/psrinfo.txt
7642026-05-19T05:20:26.426Zuploading: /tmp/debug/upstairs-info.txt (165291 bytes)
7652026-05-19T05:20:27.437Zuploaded: /tmp/debug/upstairs-info.txt