01KHF8389YVRYTWHP55H8T6T7Z: test-up-2region-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01KHF83MMZYY6DCP1H6M4C4WM2

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12026-02-14T23:54:55.105Zjob dependencies complete; ready to run (waiting for 20 m 2 s)
22026-02-14T23:55:56.343Zjob assigned to worker 01KHF98GNC38PVJZ1ER1ER3X85 [factory aws, i-092ee0ac96abcc71a] (queued for 1 m 1 s)
32026-02-14T23:56:05.002Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42026-02-14T23:56:06.588Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52026-02-14T23:56:06.588Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62026-02-14T23:56:08.826Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72026-02-14T23:56:08.826Zdownloading input: /input/build/work/bins/crutest.gz
82026-02-14T23:56:10.753Zdownloaded input: /input/build/work/bins/crutest.gz
92026-02-14T23:56:10.754Zdownloading input: /input/build/work/bins/dsc.gz
102026-02-14T23:56:11.853Zdownloaded input: /input/build/work/bins/dsc.gz
112026-02-14T23:56:11.858Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122026-02-14T23:56:11.885Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132026-02-14T23:56:11.886Zdownloading input: /input/build/work/scripts/test_ds.sh
142026-02-14T23:56:11.911Zdownloaded input: /input/build/work/scripts/test_ds.sh
152026-02-14T23:56:11.912Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162026-02-14T23:56:11.939Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172026-02-14T23:56:11.939Zdownloading input: /input/build/work/scripts/test_repair.sh
182026-02-14T23:56:11.969Zdownloaded input: /input/build/work/scripts/test_repair.sh
192026-02-14T23:56:11.969Zdownloading input: /input/build/work/scripts/test_replay.sh
202026-02-14T23:56:12.003Zdownloaded input: /input/build/work/scripts/test_replay.sh
212026-02-14T23:56:12.003Zdownloading input: /input/build/work/scripts/test_up.sh
222026-02-14T23:56:12.040Zdownloaded input: /input/build/work/scripts/test_up.sh
232026-02-14T23:56:12.040Zdownloading input: /input/build/work/scripts/upstairs_info.d
242026-02-14T23:56:12.078Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252026-02-14T23:56:12.083Zdownloading input: /input/build/tmp/cargo-test-out.log
262026-02-14T23:56:12.800Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272026-02-14T23:56:12.800Zstarting task 0: "setup"
282026-02-14T23:56:12.805Z++ uname -s
292026-02-14T23:56:12.805Z+ kern=SunOS
302026-02-14T23:56:12.805Z+ build_user=build
312026-02-14T23:56:12.805Z+ build_uid=12345
322026-02-14T23:56:12.805Z+ work_dir=/work
332026-02-14T23:56:12.805Z+ input_dir=/input
342026-02-14T23:56:12.805Z+ [[ 0 == 12345 ]]
352026-02-14T23:56:12.812Z+ case "$kern" in
362026-02-14T23:56:12.812Z+ groupadd -g 12345 build
372026-02-14T23:56:12.812Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382026-02-14T23:56:14.816Z+ zfs create -o mountpoint=/work rpool/work
392026-02-14T23:56:15.173Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402026-02-14T23:56:15.178Z+ home_fs=zfs
412026-02-14T23:56:15.178Z+ [[ zfs == autofs ]]
422026-02-14T23:56:15.178Z+ mkdir -p /home/build
432026-02-14T23:56:15.178Z+ chown build:build /home/build /work
442026-02-14T23:56:17.180Z+ chmod 0700 /home/build /work
452026-02-14T23:56:17.186Zprocess exited: duration 4387 ms, exit code 0
 
462026-02-14T23:56:17.196Zstarting task 1: "authentication"
472026-02-14T23:56:17.220Zprocess exited: duration 22 ms, exit code 0
 
482026-02-14T23:56:17.230Zstarting task 2: "build"
492026-02-14T23:56:17.234Z+ banner cores
502026-02-14T23:56:17.238Z
512026-02-14T23:56:17.238Z #### #### ##### ###### ####
522026-02-14T23:56:17.239Z # # # # # # # #
532026-02-14T23:56:17.239Z # # # # # ##### ####
542026-02-14T23:56:17.239Z # # # ##### # #
552026-02-14T23:56:17.239Z # # # # # # # # #
562026-02-14T23:56:17.239Z #### #### # # ###### ####
572026-02-14T23:56:17.239Z
582026-02-14T23:56:17.239Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
592026-02-14T23:56:17.247Z+ banner unpack
602026-02-14T23:56:17.251Z
612026-02-14T23:56:17.251Z # # # # ##### ## #### # #
622026-02-14T23:56:17.252Z # # ## # # # # # # # # #
632026-02-14T23:56:17.252Z # # # # # # # # # # ####
642026-02-14T23:56:17.252Z # # # # # ##### ###### # # #
652026-02-14T23:56:17.252Z # # # ## # # # # # # #
662026-02-14T23:56:17.252Z #### # # # # # #### # #
672026-02-14T23:56:17.252Z
682026-02-14T23:56:17.252Z+ mkdir -p /var/tmp/bins
692026-02-14T23:56:17.257Z+ for t in "$input/bins/"*.gz
702026-02-14T23:56:17.257Z++ basename /input/build/work/bins/crucible-downstairs.gz
712026-02-14T23:56:17.258Z+ b=crucible-downstairs.gz
722026-02-14T23:56:17.258Z+ b=crucible-downstairs
732026-02-14T23:56:17.258Z+ gunzip
742026-02-14T23:56:20.240Z+ chmod +x /var/tmp/bins/crucible-downstairs
752026-02-14T23:56:20.244Z+ for t in "$input/bins/"*.gz
762026-02-14T23:56:20.245Z++ basename /input/build/work/bins/crucible-hammer.gz
772026-02-14T23:56:20.249Z+ b=crucible-hammer.gz
782026-02-14T23:56:20.249Z+ b=crucible-hammer
792026-02-14T23:56:20.249Z+ gunzip
802026-02-14T23:56:23.086Z+ chmod +x /var/tmp/bins/crucible-hammer
812026-02-14T23:56:23.090Z+ for t in "$input/bins/"*.gz
822026-02-14T23:56:23.090Z++ basename /input/build/work/bins/crutest.gz
832026-02-14T23:56:23.094Z+ b=crutest.gz
842026-02-14T23:56:23.094Z+ b=crutest
852026-02-14T23:56:23.094Z+ gunzip
862026-02-14T23:56:26.049Z+ chmod +x /var/tmp/bins/crutest
872026-02-14T23:56:26.053Z+ for t in "$input/bins/"*.gz
882026-02-14T23:56:26.053Z++ basename /input/build/work/bins/dsc.gz
892026-02-14T23:56:26.053Z+ b=dsc.gz
902026-02-14T23:56:26.053Z+ b=dsc
912026-02-14T23:56:26.053Z+ gunzip
922026-02-14T23:56:27.517Z+ chmod +x /var/tmp/bins/dsc
932026-02-14T23:56:27.521Z+ export BINDIR=/var/tmp/bins
942026-02-14T23:56:27.521Z+ BINDIR=/var/tmp/bins
952026-02-14T23:56:27.521Z+ jobpid=1157
962026-02-14T23:56:27.521Z+ echo 'Setup debug logging'
972026-02-14T23:56:27.521ZSetup debug logging
982026-02-14T23:56:27.521Z+ mkdir /tmp/debug
992026-02-14T23:56:27.521Z+ sleep 3600
1002026-02-14T23:56:27.525Z+ psrinfo -v
1012026-02-14T23:56:27.529Z+ df -h
1022026-02-14T23:56:27.534Z+ prstat -d d -mLc 1
1032026-02-14T23:56:27.538Z+ iostat -T d -xn 1
1042026-02-14T23:56:27.538Z+ mpstat -T d 1
1052026-02-14T23:56:27.538Z+ vmstat -T d -p 1
1062026-02-14T23:56:27.538Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1072026-02-14T23:56:27.538Z+ banner test_up_2r_encrypted
1082026-02-14T23:56:27.538Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1092026-02-14T23:56:27.542Z #####
1102026-02-14T23:56:27.542Z ##### ###### #### ##### # # ##### # # #####
1112026-02-14T23:56:27.542Z # # # # # # # # # # #
1122026-02-14T23:56:27.542Z # ##### #### # # # # # ##### # #
1132026-02-14T23:56:27.542Z # # # # # # ##### # #####
1142026-02-14T23:56:27.542Z # # # # # # # # # # #
1152026-02-14T23:56:27.542Z # ###### #### # ####### #### # ####### ####### # #
1162026-02-14T23:56:27.543Z
1172026-02-14T23:56:27.543Z+ ptime -m bash /input/build/work/scripts/test_up.sh -r 2 -N encrypted
1182026-02-14T23:56:27.552Z/input/build/work
1192026-02-14T23:56:27.564ZUsing 2 region sets
1202026-02-14T23:56:27.564ZTurn off color for downstairs dump
1212026-02-14T23:56:27.579ZUpstairs using key: xiNLEoEyrMJswPcDHAlO/ZXbgA8dEjXzTcoPBeESU+o=
1222026-02-14T23:56:27.583Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1232026-02-14T23:56:27.583ZCreating 6 downstairs regions
1242026-02-14T23:56:27.935ZStarting 6 downstairs
1252026-02-14T23:56:27.939Zdsc started at PID: 1205
1262026-02-14T23:56:33.292ZDisable automatic restart on all downstairs
1272026-02-14T23:56:33.349Z
1282026-02-14T23:56:33.349ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1292026-02-14T23:56:33.353ZRunning test: span
1302026-02-14T23:56:37.547ZCompleted test: span
1312026-02-14T23:56:42.554ZRunning test: big
1322026-02-14T23:56:47.138ZCompleted test: big
1332026-02-14T23:56:52.145ZRunning test: dep
1342026-02-14T23:57:25.782ZCompleted test: dep
1352026-02-14T23:57:30.789ZRunning test: balloon
1362026-02-15T00:01:45.527ZCompleted test: balloon
1372026-02-15T00:01:50.534ZRunning test: deactivate
1382026-02-15T00:01:55.212ZCompleted test: deactivate
1392026-02-15T00:02:00.219ZRunning hammer
1402026-02-15T00:02:18.419ZRun repair tests
1412026-02-15T00:02:18.424Z/var/tmp/bins/crutest fill -g 31 -q --key xiNLEoEyrMJswPcDHAlO/ZXbgA8dEjXzTcoPBeESU+o= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1422026-02-15T00:02:18.483Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.837149919Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"task":"crutest"}
1432026-02-15T00:02:18.488Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.837788912Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"task":"crutest"}
1442026-02-15T00:02:18.488Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.83962833Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"task":"crutest"}
1452026-02-15T00:02:18.489Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.839983548Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248}
1462026-02-15T00:02:18.489Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.840045609Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248}
1472026-02-15T00:02:18.489Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.84007436Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248}
1482026-02-15T00:02:18.489Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.840112371Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248}
1492026-02-15T00:02:18.489Z{"msg":"Crucible 6d52b721-14ca-4dd2-b468-0b35ec8e3969 has session id: 8d5286d1-9398-45a4-a111-58991117b1b4","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.840661272Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1502026-02-15T00:02:18.489Z{"msg":"Upstairs opts: Upstairs UUID: 6d52b721-14ca-4dd2-b468-0b35ec8e3969, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.840760594Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1512026-02-15T00:02:18.489Z{"msg":"Crucible stats registered with UUID: 6d52b721-14ca-4dd2-b468-0b35ec8e3969","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.840791805Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1522026-02-15T00:02:18.489Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:16.840808775Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1532026-02-15T00:02:18.489Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.84102496Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1542026-02-15T00:02:18.490Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.841084211Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1552026-02-15T00:02:18.490Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.841126022Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1562026-02-15T00:02:18.498Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.842937999Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"task":"crutest"}
1572026-02-15T00:02:18.499Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.843060472Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248}
1582026-02-15T00:02:18.499Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.843133674Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248}
1592026-02-15T00:02:18.499Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.843172724Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248}
1602026-02-15T00:02:18.499Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.843218195Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248}
1612026-02-15T00:02:18.499Z{"msg":"Crucible 6d52b721-14ca-4dd2-b468-0b35ec8e3969 has session id: 2aabf6ae-008b-457b-83e6-acca166d29ac","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.843571032Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
1622026-02-15T00:02:18.499Z{"msg":"Upstairs opts: Upstairs UUID: 6d52b721-14ca-4dd2-b468-0b35ec8e3969, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.843656184Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
1632026-02-15T00:02:18.499Z{"msg":"Crucible stats registered with UUID: 6d52b721-14ca-4dd2-b468-0b35ec8e3969","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.843702815Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
1642026-02-15T00:02:18.500Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:16.843735676Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
1652026-02-15T00:02:18.500Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.843773077Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
1662026-02-15T00:02:18.500Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.843816908Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
1672026-02-15T00:02:18.500Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.843858238Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
1682026-02-15T00:02:18.500Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.844355769Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"task":"crutest"}
1692026-02-15T00:02:18.500Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.84440105Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248}
1702026-02-15T00:02:18.500Z{"msg":"6d52b721-14ca-4dd2-b468-0b35ec8e3969 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.84441909Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1712026-02-15T00:02:18.500Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.84443687Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"client":"0","":"downstairs","session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1722026-02-15T00:02:18.501Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.844468581Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"client":"1","":"downstairs","session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1732026-02-15T00:02:18.501Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.844507102Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"client":"2","":"downstairs","session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1742026-02-15T00:02:18.501Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.844536223Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1752026-02-15T00:02:18.501Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.844579264Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1762026-02-15T00:02:18.501Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.844609614Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1772026-02-15T00:02:18.502Z{"msg":"ds_connection connected from Ok(127.0.0.1:49282)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.844635685Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1782026-02-15T00:02:18.502Z{"msg":"ds_connection connected from Ok(127.0.0.1:36031)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.844691446Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1792026-02-15T00:02:18.502Z{"msg":"ds_connection connected from Ok(127.0.0.1:59319)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.844922821Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1802026-02-15T00:02:18.502Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.846051734Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"client":"2","":"downstairs","session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1812026-02-15T00:02:18.502Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.846167086Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"client":"1","":"downstairs","session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1822026-02-15T00:02:18.502Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.846226938Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"client":"0","":"downstairs","session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1832026-02-15T00:02:18.502Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.846426792Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1842026-02-15T00:02:18.502Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.846543544Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1852026-02-15T00:02:18.502Z{"msg":"[0]R flush_numbers: [4004, 4004, 4004, 4004, 4004]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.846582415Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"downstairs","":"downstairs","session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1862026-02-15T00:02:18.503Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.846609246Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"downstairs","":"downstairs","session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1872026-02-15T00:02:18.503Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.846628936Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"downstairs","":"downstairs","session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1882026-02-15T00:02:18.503Z{"msg":"[1]R flush_numbers: [4004, 4004, 4004, 4004, 4004]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.846653507Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"downstairs","":"downstairs","session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1892026-02-15T00:02:18.503Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.846671727Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"downstairs","":"downstairs","session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1902026-02-15T00:02:18.503Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.846695568Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"downstairs","":"downstairs","session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1912026-02-15T00:02:18.503Z{"msg":"[2]R flush_numbers: [4004, 4004, 4004, 4004, 4004]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.846713678Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"downstairs","":"downstairs","session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1922026-02-15T00:02:18.503Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.846737308Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"downstairs","":"downstairs","session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1932026-02-15T00:02:18.503Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.846755289Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"downstairs","":"downstairs","session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1942026-02-15T00:02:18.504Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.846778029Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"downstairs","":"downstairs","session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1952026-02-15T00:02:18.504Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.84679549Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"downstairs","":"downstairs","session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1962026-02-15T00:02:18.504Z{"msg":"Next flush: 4005","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.84681851Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"downstairs","":"downstairs","session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1972026-02-15T00:02:18.504Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.84683515Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"downstairs","":"downstairs","session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1982026-02-15T00:02:18.504Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.846858841Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
1992026-02-15T00:02:18.504Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.846874771Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"client":"0","":"downstairs","session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
2002026-02-15T00:02:18.505Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.846897872Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"client":"1","":"downstairs","session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
2012026-02-15T00:02:18.505Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.846914792Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"client":"2","":"downstairs","session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
2022026-02-15T00:02:18.505Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.846971333Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
2032026-02-15T00:02:18.505Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.847062735Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
2042026-02-15T00:02:18.505Z{"msg":"6d52b721-14ca-4dd2-b468-0b35ec8e3969 is now active with session: 8d5286d1-9398-45a4-a111-58991117b1b4","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.847089776Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
2052026-02-15T00:02:18.505Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.847137597Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248}
2062026-02-15T00:02:18.505Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.847173027Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"session_id":"8d5286d1-9398-45a4-a111-58991117b1b4"}
2072026-02-15T00:02:18.505Z{"msg":"Activated sub_volume 6d52b721-14ca-4dd2-b468-0b35ec8e3969","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.84731064Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248}
2082026-02-15T00:02:18.506Z{"msg":"6d52b721-14ca-4dd2-b468-0b35ec8e3969 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.847338201Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2092026-02-15T00:02:18.506Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.847361211Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"client":"0","":"downstairs","session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2102026-02-15T00:02:18.506Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.847380472Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248}
2112026-02-15T00:02:18.506Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.847394122Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"client":"1","":"downstairs","session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2122026-02-15T00:02:18.506Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.847418603Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"client":"2","":"downstairs","session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2132026-02-15T00:02:18.506Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.847435423Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2142026-02-15T00:02:18.506Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.847452853Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2152026-02-15T00:02:18.506Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.847470544Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2162026-02-15T00:02:18.506Z{"msg":"ds_connection connected from Ok(127.0.0.1:45634)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.847496324Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2172026-02-15T00:02:18.506Z{"msg":"ds_connection connected from Ok(127.0.0.1:46917)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.847526905Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2182026-02-15T00:02:18.507Z{"msg":"ds_connection connected from Ok(127.0.0.1:43631)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.847563055Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2192026-02-15T00:02:18.507Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.84825802Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"client":"1","":"downstairs","session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2202026-02-15T00:02:18.507Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.848310841Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"client":"2","":"downstairs","session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2212026-02-15T00:02:18.507Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.848388983Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"client":"0","":"downstairs","session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2222026-02-15T00:02:18.507Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.848625097Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2232026-02-15T00:02:18.507Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.848675149Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2242026-02-15T00:02:18.513Z{"msg":"[0]R flush_numbers: [5324, 5324, 5312, 5323, 5315]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.848705749Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"downstairs","":"downstairs","session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2252026-02-15T00:02:18.513Z{"msg":"[0]R generation: [14, 14, 11, 14, 12]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.84875099Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"downstairs","":"downstairs","session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2262026-02-15T00:02:18.513Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.848797271Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"downstairs","":"downstairs","session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2272026-02-15T00:02:18.513Z{"msg":"[1]R flush_numbers: [5324, 5324, 5312, 5323, 5315]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.848833672Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"downstairs","":"downstairs","session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2282026-02-15T00:02:18.513ZWait for a query_work_queue command to finish before sending IO
2292026-02-15T00:02:18.513Z{"msg":"[1]R generation: [14, 14, 11, 14, 12]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.848884663Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"downstairs","":"downstairs","session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2302026-02-15T00:02:18.514Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.848924094Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"downstairs","":"downstairs","session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2312026-02-15T00:02:18.514Z{"msg":"[2]R flush_numbers: [5324, 5324, 5312, 5323, 5315]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.848968925Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"downstairs","":"downstairs","session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2322026-02-15T00:02:18.514Z{"msg":"[2]R generation: [14, 14, 11, 14, 12]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.849013026Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"downstairs","":"downstairs","session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2332026-02-15T00:02:18.514Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.849061127Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"downstairs","":"downstairs","session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2342026-02-15T00:02:18.514Z{"msg":"Max found gen is 15","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.849102638Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"downstairs","":"downstairs","session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2352026-02-15T00:02:18.514Z{"msg":"Generation requested: 31 >= found:15","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.849130388Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"downstairs","":"downstairs","session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2362026-02-15T00:02:18.514Z{"msg":"Next flush: 5325","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.849167379Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"downstairs","":"downstairs","session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2372026-02-15T00:02:18.515Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.849186409Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"":"downstairs","":"downstairs","session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2382026-02-15T00:02:18.515Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.84921152Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2392026-02-15T00:02:18.515Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.84923058Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"client":"0","":"downstairs","session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2402026-02-15T00:02:18.515Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.849266071Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"client":"1","":"downstairs","session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2412026-02-15T00:02:18.515Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.849291891Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"client":"2","":"downstairs","session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2422026-02-15T00:02:18.515Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.849328402Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2432026-02-15T00:02:18.515Z{"msg":"Disk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
2442026-02-15T00:02:18.515ZSet Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.849357013Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2452026-02-15T00:02:18.515Z{"msg":"Fill test
2462026-02-15T00:02:18.515Z6d52b721-14ca-4dd2-b468-0b35ec8e3969 is now active with session: 2aabf6ae-008b-457b-83e6-acca166d29ac","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.849377493Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2472026-02-15T00:02:18.515Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.849399314Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248,"session_id":"2aabf6ae-008b-457b-83e6-acca166d29ac"}
2482026-02-15T00:02:18.515Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.849415634Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248}
2492026-02-15T00:02:18.515Z{"msg":"Activated sub_volume 6d52b721-14ca-4dd2-b468-0b35ec8e3969","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:16.849435925Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248}
2502026-02-15T00:02:18.538ZRead and Verify all blocks (0..100 range:false)
2512026-02-15T00:02:18.617ZWrote out file "/var/tmp/test_up-build/verify_file"
2522026-02-15T00:02:18.617ZCLIENT: Tests done. All submitted work has been ACK'd
2532026-02-15T00:02:18.621Z----------------------------------------------------------------
2542026-02-15T00:02:18.621Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2552026-02-15T00:02:18.621ZStates: Active Active Active
2562026-02-15T00:02:18.622ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2572026-02-15T00:02:18.622Z 1002 Acked Read 50 Done Done Done false
2582026-02-15T00:02:18.622Z STATES DS:0 DS:1 DS:2 TOTAL
2592026-02-15T00:02:18.622Z Sent 0 0 0 0
2602026-02-15T00:02:18.622Z Done 1 1 1 3
2612026-02-15T00:02:18.622Z Skipped 0 0 0 0
2622026-02-15T00:02:18.622Z Error 0 0 0 0
2632026-02-15T00:02:18.622ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2642026-02-15T00:02:18.622ZDownstairs last five completed: 1001 1000
2652026-02-15T00:02:18.622ZUpstairs last five completed: 1002 1001 1000
2662026-02-15T00:02:18.623Z----------------------------------------------------------------
2672026-02-15T00:02:18.623Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2682026-02-15T00:02:18.623ZStates: Active Active Active
2692026-02-15T00:02:18.623ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2702026-02-15T00:02:18.623Z 1002 Acked Read 50 Done Done Done false
2712026-02-15T00:02:18.623Z STATES DS:0 DS:1 DS:2 TOTAL
2722026-02-15T00:02:18.623Z Sent 0 0 0 0
2732026-02-15T00:02:18.623Z Done 1 1 1 3
2742026-02-15T00:02:18.623Z Skipped 0 0 0 0
2752026-02-15T00:02:18.623Z Error 0 0 0 0
2762026-02-15T00:02:18.623ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2772026-02-15T00:02:18.623ZDownstairs last five completed: 1001 1000
2782026-02-15T00:02:18.623ZUpstairs last five completed: 1002 1001 1000
2792026-02-15T00:02:18.623ZCLIENT: Up:0 ds:2 act:6
2802026-02-15T00:02:22.617Z----------------------------------------------------------------
2812026-02-15T00:02:22.617Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2822026-02-15T00:02:22.617ZDownstairs last five completed: 1003 1002 1001 1000
2832026-02-15T00:02:22.617ZUpstairs last five completed: 1003 1002 1001 1000
2842026-02-15T00:02:22.621Z----------------------------------------------------------------
2852026-02-15T00:02:22.621Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2862026-02-15T00:02:22.621ZDownstairs last five completed: 1003 1002 1001 1000
2872026-02-15T00:02:22.621ZUpstairs last five completed: 1003 1002 1001 1000
2882026-02-15T00:02:22.621ZCLIENT: Up:0 ds:0 act:6
2892026-02-15T00:02:22.621ZCLIENT: All crucible jobs finished, exiting program
2902026-02-15T00:02:22.621Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:20.972343608Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1248}
2912026-02-15T00:02:22.630ZRepair setup passed
2922026-02-15T00:02:22.634ZCopy the region for /var/tmp/test_up-build/8830
2932026-02-15T00:02:22.634Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2942026-02-15T00:02:22.639Z/var/tmp/bins/crutest fill -g 32 -q --key xiNLEoEyrMJswPcDHAlO/ZXbgA8dEjXzTcoPBeESU+o= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2952026-02-15T00:02:22.695Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.049316898Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"task":"crutest"}
2962026-02-15T00:02:22.699Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.049880849Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"task":"crutest"}
2972026-02-15T00:02:22.699Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.051721618Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"task":"crutest"}
2982026-02-15T00:02:22.699Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.052017094Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254}
2992026-02-15T00:02:22.699Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.052087365Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254}
3002026-02-15T00:02:22.699Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.052130546Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254}
3012026-02-15T00:02:22.699Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.052168787Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254}
3022026-02-15T00:02:22.699Z{"msg":"Crucible 65cb4b1c-7e27-40d6-ac63-40f237cc2d02 has session id: 47c3288f-b58a-404f-ae85-d1ed31b62386","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.052732249Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3032026-02-15T00:02:22.699Z{"msg":"Upstairs opts: Upstairs UUID: 65cb4b1c-7e27-40d6-ac63-40f237cc2d02, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.05279628Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3042026-02-15T00:02:22.699Z{"msg":"Crucible stats registered with UUID: 65cb4b1c-7e27-40d6-ac63-40f237cc2d02","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.052843951Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3052026-02-15T00:02:22.699Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:21.052863781Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3062026-02-15T00:02:22.699Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.053015884Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3072026-02-15T00:02:22.699Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.053067546Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3082026-02-15T00:02:22.699Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.053111117Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3092026-02-15T00:02:22.705Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.055074077Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"task":"crutest"}
3102026-02-15T00:02:22.705Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.055181569Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254}
3112026-02-15T00:02:22.705Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.055229421Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254}
3122026-02-15T00:02:22.705Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.055244791Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254}
3132026-02-15T00:02:22.705Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.055266581Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254}
3142026-02-15T00:02:22.705Z{"msg":"Crucible 65cb4b1c-7e27-40d6-ac63-40f237cc2d02 has session id: 66d17ea8-e2e5-4f8f-b692-3401afddf3a3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.05566934Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3152026-02-15T00:02:22.705Z{"msg":"Upstairs opts: Upstairs UUID: 65cb4b1c-7e27-40d6-ac63-40f237cc2d02, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.055740911Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3162026-02-15T00:02:22.705Z{"msg":"Crucible stats registered with UUID: 65cb4b1c-7e27-40d6-ac63-40f237cc2d02","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.055787532Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3172026-02-15T00:02:22.705Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:21.055828903Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3182026-02-15T00:02:22.705Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.055860544Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3192026-02-15T00:02:22.705Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.055903855Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3202026-02-15T00:02:22.705Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.055939435Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3212026-02-15T00:02:22.705Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.056503397Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"task":"crutest"}
3222026-02-15T00:02:22.705Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.056549058Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254}
3232026-02-15T00:02:22.705Z{"msg":"65cb4b1c-7e27-40d6-ac63-40f237cc2d02 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.056579159Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3242026-02-15T00:02:22.705Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.05662097Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"client":"0","":"downstairs","session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3252026-02-15T00:02:22.705Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.056687871Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"client":"1","":"downstairs","session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3262026-02-15T00:02:22.705Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.056800343Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"client":"2","":"downstairs","session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3272026-02-15T00:02:22.706Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.056861915Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3282026-02-15T00:02:22.706Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.056920316Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3292026-02-15T00:02:22.706Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.05711312Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3302026-02-15T00:02:22.706Z{"msg":"ds_connection connected from Ok(127.0.0.1:53584)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.057185611Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3312026-02-15T00:02:22.706Z{"msg":"ds_connection connected from Ok(127.0.0.1:63995)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.057237032Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3322026-02-15T00:02:22.706Z{"msg":"ds_connection connected from Ok(127.0.0.1:35088)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.057267183Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3332026-02-15T00:02:22.706Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.058144221Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"client":"1","":"downstairs","session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3342026-02-15T00:02:22.706Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.058180382Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"client":"0","":"downstairs","session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3352026-02-15T00:02:22.706Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.058202372Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"client":"2","":"downstairs","session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3362026-02-15T00:02:22.706Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.058541019Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3372026-02-15T00:02:22.706Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.0585768Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3382026-02-15T00:02:22.706Z{"msg":"[0]R flush_numbers: [4005, 4005, 4005, 4005, 4005]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.058595311Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"downstairs","":"downstairs","session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3392026-02-15T00:02:22.706Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.058631371Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"downstairs","":"downstairs","session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3402026-02-15T00:02:22.706Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.058667792Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"downstairs","":"downstairs","session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3412026-02-15T00:02:22.706Z{"msg":"[1]R flush_numbers: [4005, 4005, 4005, 4005, 4005]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.058708373Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"downstairs","":"downstairs","session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3422026-02-15T00:02:22.706Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.058751954Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"downstairs","":"downstairs","session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3432026-02-15T00:02:22.706Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.058795425Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"downstairs","":"downstairs","session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3442026-02-15T00:02:22.706Z{"msg":"[2]R flush_numbers: [4005, 4005, 4005, 4005, 4005]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.058839285Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"downstairs","":"downstairs","session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3452026-02-15T00:02:22.706Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.058882017Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"downstairs","":"downstairs","session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3462026-02-15T00:02:22.706Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.058949188Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"downstairs","":"downstairs","session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3472026-02-15T00:02:22.711Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.058982579Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"downstairs","":"downstairs","session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3482026-02-15T00:02:22.711Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.05904916Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"downstairs","":"downstairs","session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3492026-02-15T00:02:22.711Z{"msg":"Next flush: 4006","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.059099231Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"downstairs","":"downstairs","session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3502026-02-15T00:02:22.711Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.059261714Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"downstairs","":"downstairs","session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3512026-02-15T00:02:22.712Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.059330476Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3522026-02-15T00:02:22.712Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.059391117Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"client":"0","":"downstairs","session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3532026-02-15T00:02:22.712Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.059436528Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"client":"1","":"downstairs","session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3542026-02-15T00:02:22.712Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.059495249Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"client":"2","":"downstairs","session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3552026-02-15T00:02:22.712Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.0595584Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3562026-02-15T00:02:22.712Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.059618832Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3572026-02-15T00:02:22.712Z{"msg":"65cb4b1c-7e27-40d6-ac63-40f237cc2d02 is now active with session: 47c3288f-b58a-404f-ae85-d1ed31b62386","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.059665033Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3582026-02-15T00:02:22.712Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.059720594Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"session_id":"47c3288f-b58a-404f-ae85-d1ed31b62386"}
3592026-02-15T00:02:22.712Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.059751135Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254}
3602026-02-15T00:02:22.712Z{"msg":"Activated sub_volume 65cb4b1c-7e27-40d6-ac63-40f237cc2d02","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.059776185Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254}
3612026-02-15T00:02:22.712Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.059812546Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254}
3622026-02-15T00:02:22.712Z{"msg":"65cb4b1c-7e27-40d6-ac63-40f237cc2d02 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.059832216Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3632026-02-15T00:02:22.712Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.059861357Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"client":"0","":"downstairs","session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3642026-02-15T00:02:22.712Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.059878847Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"client":"1","":"downstairs","session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3652026-02-15T00:02:22.712Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.059895827Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"client":"2","":"downstairs","session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3662026-02-15T00:02:22.712Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.059919578Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3672026-02-15T00:02:22.712Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.059937118Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3682026-02-15T00:02:22.712Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.059961419Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3692026-02-15T00:02:22.712Z{"msg":"ds_connection connected from Ok(127.0.0.1:41603)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.059979519Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3702026-02-15T00:02:22.712Z{"msg":"ds_connection connected from Ok(127.0.0.1:48205)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.06001041Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3712026-02-15T00:02:22.713Z{"msg":"ds_connection connected from Ok(127.0.0.1:55528)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.060043321Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3722026-02-15T00:02:22.713Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.060345977Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"client":"1","":"downstairs","session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3732026-02-15T00:02:22.713Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.060405338Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"client":"2","":"downstairs","session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3742026-02-15T00:02:22.713Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.060431259Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"client":"0","":"downstairs","session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3752026-02-15T00:02:22.713Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.060688204Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3762026-02-15T00:02:22.713Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.060740815Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3772026-02-15T00:02:22.713Z{"msg":"[0]R flush_numbers: [5325, 5325, 5325, 5325, 5325]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.060784006Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"downstairs","":"downstairs","session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3782026-02-15T00:02:22.713Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.060827177Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"downstairs","":"downstairs","session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3792026-02-15T00:02:22.713Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.060873258Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"downstairs","":"downstairs","session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3802026-02-15T00:02:22.713Z{"msg":"[1]R flush_numbers: [5325, 5325, 5325, 5325, 5325]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.060920289Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"downstairs","":"downstairs","session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3812026-02-15T00:02:22.713ZWait for a query_work_queue command to finish before sending IO
3822026-02-15T00:02:22.713Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.06096932Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"downstairs","":"downstairs","session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3832026-02-15T00:02:22.713Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.061012611Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"downstairs","":"downstairs","session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3842026-02-15T00:02:22.713Z{"msg":"[2]R flush_numbers: [5325, 5325, 5325, 5325, 5325]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.061048502Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"downstairs","":"downstairs","session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3852026-02-15T00:02:22.713Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.061096932Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"downstairs","":"downstairs","session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3862026-02-15T00:02:22.713Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.061118793Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"downstairs","":"downstairs","session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3872026-02-15T00:02:22.713Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.061137493Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"downstairs","":"downstairs","session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3882026-02-15T00:02:22.713Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.061177404Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"downstairs","":"downstairs","session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3892026-02-15T00:02:22.713Z{"msg":"Next flush: 5326","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.061219165Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"downstairs","":"downstairs","session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3902026-02-15T00:02:22.714Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.061265426Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"":"downstairs","":"downstairs","session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3912026-02-15T00:02:22.714Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.061311007Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3922026-02-15T00:02:22.714Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.061352558Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"client":"0","":"downstairs","session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3932026-02-15T00:02:22.714Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.061396749Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"client":"1","":"downstairs","session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3942026-02-15T00:02:22.714Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.06143889Z","hostname"Disk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
3952026-02-15T00:02:22.714Z:"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"client":"2","":"downstairs","session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3962026-02-15T00:02:22.714Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.0614741Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3972026-02-15T00:02:22.714Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.061501811Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3982026-02-15T00:02:22.714Z{"msg":"65cb4b1c-7e27-40d6-ac63-40f237cc2d02 is now active with session: 66d17ea8-e2e5-4f8f-b692-3401afddf3a3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.061520791Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
3992026-02-15T00:02:22.714Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.061543322Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254,"session_id":"66d17ea8-e2e5-4f8f-b692-3401afddf3a3"}
4002026-02-15T00:02:22.714Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.061559942Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254}
4012026-02-15T00:02:22.714Z{"msg":"Activated sub_volume 65cb4b1c-7e27-40d6-ac63-40f237cc2d02","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.061574413Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254}
4022026-02-15T00:02:22.714ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4032026-02-15T00:02:22.714ZFill test
4042026-02-15T00:02:22.754ZRead and Verify all blocks (0..100 range:false)
4052026-02-15T00:02:22.830ZWrote out file "/var/tmp/test_up-build/verify_file"
4062026-02-15T00:02:22.830ZCLIENT: Tests done. All submitted work has been ACK'd
4072026-02-15T00:02:22.831Z----------------------------------------------------------------
4082026-02-15T00:02:22.837Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4092026-02-15T00:02:22.837ZStates: Active Active Active
4102026-02-15T00:02:22.837ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4112026-02-15T00:02:22.837Z 1002 Acked Read 50 Done Done Done false
4122026-02-15T00:02:22.838Z STATES DS:0 DS:1 DS:2 TOTAL
4132026-02-15T00:02:22.838Z Sent 0 0 0 0
4142026-02-15T00:02:22.838Z Done 1 1 1 3
4152026-02-15T00:02:22.838Z Skipped 0 0 0 0
4162026-02-15T00:02:22.839Z Error 0 0 0 0
4172026-02-15T00:02:22.839ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4182026-02-15T00:02:22.839ZDownstairs last five completed: 1001 1000
4192026-02-15T00:02:22.840ZUpstairs last five completed: 1002 1001 1000
4202026-02-15T00:02:22.840Z----------------------------------------------------------------
4212026-02-15T00:02:22.840Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4222026-02-15T00:02:22.840ZStates: Active Active Active
4232026-02-15T00:02:22.841ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4242026-02-15T00:02:22.841Z 1002 Acked Read 50 Done Done Done false
4252026-02-15T00:02:22.841Z STATES DS:0 DS:1 DS:2 TOTAL
4262026-02-15T00:02:22.841Z Sent 0 0 0 0
4272026-02-15T00:02:22.841Z Done 1 1 1 3
4282026-02-15T00:02:22.842Z Skipped 0 0 0 0
4292026-02-15T00:02:22.842Z Error 0 0 0 0
4302026-02-15T00:02:22.842ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4312026-02-15T00:02:22.842ZDownstairs last five completed: 1001 1000
4322026-02-15T00:02:22.842ZUpstairs last five completed: 1002 1001 1000
4332026-02-15T00:02:22.843ZCLIENT: Up:0 ds:2 act:6
4342026-02-15T00:02:26.832Z----------------------------------------------------------------
4352026-02-15T00:02:26.832Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4362026-02-15T00:02:26.832ZDownstairs last five completed: 1003 1002 1001 1000
4372026-02-15T00:02:26.832ZUpstairs last five completed: 1003 1002 1001 1000
4382026-02-15T00:02:26.837Z----------------------------------------------------------------
4392026-02-15T00:02:26.837Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4402026-02-15T00:02:26.837ZDownstairs last five completed: 1003 1002 1001 1000
4412026-02-15T00:02:26.837ZUpstairs last five completed: 1003 1002 1001 1000
4422026-02-15T00:02:26.837ZCLIENT: Up:0 ds:0 act:6
4432026-02-15T00:02:26.837ZCLIENT: All crucible jobs finished, exiting program
4442026-02-15T00:02:26.837Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:25.185675311Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1254}
4452026-02-15T00:02:26.842ZRepair part 1 passed
4462026-02-15T00:02:26.842Z
4472026-02-15T00:02:26.843ZKill the current downstairs
4482026-02-15T00:02:27.009ZDownstairs 2 stopped
4492026-02-15T00:02:27.009Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
4502026-02-15T00:02:27.013ZNow put back the original so we have a mismatch
4512026-02-15T00:02:27.013Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
4522026-02-15T00:02:27.018ZRestart downstairs with old directory
4532026-02-15T00:02:27.071Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4542026-02-15T00:02:27.090ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4552026-02-15T00:02:27.095Z 0 000-009 32 32 31 4006 4006 4005 F F F <---
4562026-02-15T00:02:27.095Z 1 010-019 32 32 31 4006 4006 4005 F F F <---
4572026-02-15T00:02:27.095Z 2 020-029 32 32 31 4006 4006 4005 F F F <---
4582026-02-15T00:02:27.095Z 3 030-039 32 32 31 4006 4006 4005 F F F <---
4592026-02-15T00:02:27.095Z 4 040-049 32 32 31 4006 4006 4005 F F F <---
4602026-02-15T00:02:27.095ZMax gen: 32, Max flush: 4006
4612026-02-15T00:02:27.095ZError: Difference in extent metadata found!
4622026-02-15T00:02:27.096Zdump test found error as expected
4632026-02-15T00:02:27.096Z
4642026-02-15T00:02:27.096Z
4652026-02-15T00:02:27.096Z/var/tmp/bins/crutest verify --range -g 33 -q --key xiNLEoEyrMJswPcDHAlO/ZXbgA8dEjXzTcoPBeESU+o= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
4662026-02-15T00:02:27.153Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.506626432Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"task":"crutest"}
4672026-02-15T00:02:27.157Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.507311606Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"task":"crutest"}
4682026-02-15T00:02:27.157Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.509140075Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"task":"crutest"}
4692026-02-15T00:02:27.157Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.509453771Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262}
4702026-02-15T00:02:27.157Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.509511582Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262}
4712026-02-15T00:02:27.157Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.509539783Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262}
4722026-02-15T00:02:27.158Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.509567213Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262}
4732026-02-15T00:02:27.158Z{"msg":"Crucible 1cd00e75-c45b-48d8-af30-1ebbb210ab0b has session id: 754c3e9b-206a-479e-ba31-fd19093553bc","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.510185316Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
4742026-02-15T00:02:27.158Z{"msg":"Upstairs opts: Upstairs UUID: 1cd00e75-c45b-48d8-af30-1ebbb210ab0b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.510244598Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
4752026-02-15T00:02:27.158Z{"msg":"Crucible stats registered with UUID: 1cd00e75-c45b-48d8-af30-1ebbb210ab0b","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.510268108Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
4762026-02-15T00:02:27.158Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:25.510306319Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
4772026-02-15T00:02:27.158Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.510419641Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
4782026-02-15T00:02:27.158Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.510483863Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
4792026-02-15T00:02:27.158Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.510511333Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
4802026-02-15T00:02:27.163Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.512446943Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"task":"crutest"}
4812026-02-15T00:02:27.163Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.512555886Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262}
4822026-02-15T00:02:27.163Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.512617467Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262}
4832026-02-15T00:02:27.163Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.512712139Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262}
4842026-02-15T00:02:27.163Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.51278741Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262}
4852026-02-15T00:02:27.163Z{"msg":"Crucible 1cd00e75-c45b-48d8-af30-1ebbb210ab0b has session id: 67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.513143408Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
4862026-02-15T00:02:27.163Z{"msg":"Upstairs opts: Upstairs UUID: 1cd00e75-c45b-48d8-af30-1ebbb210ab0b, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.513198169Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
4872026-02-15T00:02:27.163Z{"msg":"Crucible stats registered with UUID: 1cd00e75-c45b-48d8-af30-1ebbb210ab0b","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.51325036Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
4882026-02-15T00:02:27.164Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:25.513306251Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
4892026-02-15T00:02:27.164Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.513359462Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
4902026-02-15T00:02:27.164Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.513410063Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
4912026-02-15T00:02:27.164Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.513442964Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
4922026-02-15T00:02:27.164Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.513871443Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"task":"crutest"}
4932026-02-15T00:02:27.164Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.513917284Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262}
4942026-02-15T00:02:27.164Z{"msg":"1cd00e75-c45b-48d8-af30-1ebbb210ab0b active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.513961825Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
4952026-02-15T00:02:27.164Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.513995796Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"client":"0","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
4962026-02-15T00:02:27.164Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.514014366Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"client":"1","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
4972026-02-15T00:02:27.164Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.514044966Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"client":"2","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
4982026-02-15T00:02:27.165Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.514062457Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
4992026-02-15T00:02:27.165Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.514098667Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5002026-02-15T00:02:27.165Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.514125498Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5012026-02-15T00:02:27.165Z{"msg":"ds_connection connected from Ok(127.0.0.1:62664)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.514163119Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5022026-02-15T00:02:27.165Z{"msg":"ds_connection connected from Ok(127.0.0.1:37386)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.514191249Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5032026-02-15T00:02:27.165Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:25.5142252Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5042026-02-15T00:02:27.165Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:25.514258901Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5052026-02-15T00:02:27.165Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:25.514293642Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5062026-02-15T00:02:27.165Z{"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.514514026Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5072026-02-15T00:02:27.165Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.514562197Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5082026-02-15T00:02:27.165Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.515478476Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"client":"0","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5092026-02-15T00:02:27.165Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.515526097Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"client":"1","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5102026-02-15T00:02:27.165Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.515892005Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5112026-02-15T00:02:27.165Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:25.515929116Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5122026-02-15T00:02:37.161Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.515513956Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5132026-02-15T00:02:37.161Z{"msg":"ds_connection connected from Ok(127.0.0.1:62379)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.515644508Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5142026-02-15T00:02:37.166Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.517399745Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"client":"2","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5152026-02-15T00:02:37.166Z{"msg":"[0]R flush_numbers: [4006, 4006, 4006, 4006, 4006]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.517814853Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"downstairs","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5162026-02-15T00:02:37.166Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.517853134Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"downstairs","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5172026-02-15T00:02:37.166Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.517890095Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"downstairs","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5182026-02-15T00:02:37.166Z{"msg":"[1]R flush_numbers: [4006, 4006, 4006, 4006, 4006]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.517928756Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"downstairs","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5192026-02-15T00:02:37.166Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.517964937Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"downstairs","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5202026-02-15T00:02:37.166Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.517984617Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"downstairs","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5212026-02-15T00:02:37.166Z{"msg":"[2]R flush_numbers: [4005, 4005, 4005, 4005, 4005]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.518024378Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"downstairs","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5222026-02-15T00:02:37.166Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.518061069Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"downstairs","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5232026-02-15T00:02:37.166Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.518096779Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"downstairs","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5242026-02-15T00:02:37.166Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.5181315Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"downstairs","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5252026-02-15T00:02:37.166Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.518165851Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"downstairs","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5262026-02-15T00:02:37.166Z{"msg":"Next flush: 4007","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.518201661Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"downstairs","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5272026-02-15T00:02:37.166Z{"msg":"extent 0 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4006, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4006, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4005, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.518246833Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5282026-02-15T00:02:37.166Z{"msg":"extent:0 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4006, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4006, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4005, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.518292153Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5292026-02-15T00:02:37.166Z{"msg":"found dest for source 0 for extent at index 0 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.518329434Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5302026-02-15T00:02:37.166Z{"msg":"extent 1 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4006, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4006, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4005, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.518372065Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5312026-02-15T00:02:37.166Z{"msg":"extent:1 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4006, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4006, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4005, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.518423236Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5322026-02-15T00:02:37.166Z{"msg":"found dest for source 0 for extent at index 1 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.518463397Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5332026-02-15T00:02:37.166Z{"msg":"extent 2 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4006, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4006, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4005, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.518507858Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5342026-02-15T00:02:37.166Z{"msg":"extent:2 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4006, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4006, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4005, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.518554069Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5352026-02-15T00:02:37.166Z{"msg":"found dest for source 0 for extent at index 2 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.51859411Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5362026-02-15T00:02:37.166Z{"msg":"extent 3 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4006, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4006, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4005, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.51863629Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5372026-02-15T00:02:37.166Z{"msg":"extent:3 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4006, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4006, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4005, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.518678731Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5382026-02-15T00:02:37.166Z{"msg":"found dest for source 0 for extent at index 3 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.518715392Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5392026-02-15T00:02:37.167Z{"msg":"extent 4 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4006, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4006, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4005, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.518762503Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5402026-02-15T00:02:37.167Z{"msg":"extent:4 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4006, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4006, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4005, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.518812554Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5412026-02-15T00:02:37.167Z{"msg":"found dest for source 0 for extent at index 4 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.518852535Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5422026-02-15T00:02:37.167Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.518893336Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"client":"0","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5432026-02-15T00:02:37.167Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.518931407Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"client":"1","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5442026-02-15T00:02:37.167Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.518963767Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"client":"2","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5452026-02-15T00:02:37.167Z{"msg":"Full repair list: {ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.519017889Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"downstairs","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5462026-02-15T00:02:37.170Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 4007, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 4007, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 4007, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 4007, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 4007, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.519237763Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"downstairs","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5472026-02-15T00:02:37.170Z{"msg":"starting reconciliation 17e58c12-d7ab-489e-9a36-db43d1abcb65: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.519282804Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"downstairs","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5482026-02-15T00:02:37.170Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.519325735Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"client":"2","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5492026-02-15T00:02:37.175Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.530419706Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"client":"2","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5502026-02-15T00:02:37.179Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.534190474Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"client":"2","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5512026-02-15T00:02:37.183Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.537898981Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"client":"2","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5522026-02-15T00:02:37.186Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.541622688Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"client":"2","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5532026-02-15T00:02:37.190Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.544739443Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"downstairs","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5542026-02-15T00:02:37.190Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.544779664Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"client":"0","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5552026-02-15T00:02:37.193Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.544799354Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"client":"1","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5562026-02-15T00:02:37.193Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.544823945Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"client":"2","":"downstairs","session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5572026-02-15T00:02:37.193Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.544843345Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5582026-02-15T00:02:37.193Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.544872196Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5592026-02-15T00:02:37.193Z{"msg":"1cd00e75-c45b-48d8-af30-1ebbb210ab0b is now active with session: 754c3e9b-206a-479e-ba31-fd19093553bc","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.544903056Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"session_id":"754c3e9b-206a-479e-ba31-fd19093553bc"}
5602026-02-15T00:02:37.193Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.544921347Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262}
5612026-02-15T00:02:37.193Z{"msg":"Activated sub_volume 1cd00e75-c45b-48d8-af30-1ebbb210ab0b","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.544935487Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262}
5622026-02-15T00:02:37.193Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.544955768Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262}
5632026-02-15T00:02:37.193Z{"msg":"1cd00e75-c45b-48d8-af30-1ebbb210ab0b active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.544969958Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5642026-02-15T00:02:37.193Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.544992308Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"client":"0","":"downstairs","session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5652026-02-15T00:02:37.193Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.545022999Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"client":"1","":"downstairs","session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5662026-02-15T00:02:37.193Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.545040869Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"client":"2","":"downstairs","session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5672026-02-15T00:02:37.193Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.54506713Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5682026-02-15T00:02:37.193Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.54509788Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5692026-02-15T00:02:37.193Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.545130021Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5702026-02-15T00:02:37.193Z{"msg":"ds_connection connected from Ok(127.0.0.1:34054)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.545162462Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5712026-02-15T00:02:37.193Z{"msg":"ds_connection connected from Ok(127.0.0.1:38206)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.545196723Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5722026-02-15T00:02:37.193Z{"msg":"ds_connection connected from Ok(127.0.0.1:65438)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.545235714Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5732026-02-15T00:02:37.194Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.546105181Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"client":"1","":"downstairs","session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5742026-02-15T00:02:37.194Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.546212264Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"client":"0","":"downstairs","session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5752026-02-15T00:02:37.194Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.546261655Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"client":"2","":"downstairs","session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5762026-02-15T00:02:37.194Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.546429028Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5772026-02-15T00:02:37.194Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.546580471Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5782026-02-15T00:02:37.194Z{"msg":"[0]R flush_numbers: [5326, 5326, 5326, 5326, 5326]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.546618812Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"downstairs","":"downstairs","session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5792026-02-15T00:02:37.194Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.546657783Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"downstairs","":"downstairs","session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5802026-02-15T00:02:37.194Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.546688794Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"downstairs","":"downstairs","session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5812026-02-15T00:02:37.194Z{"msg":"[1]R flush_numbers: [5326, 5326, 5326, 5326, 5326]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.546733114Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"downstairs","":"downstairs","session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5822026-02-15T00:02:37.194Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.546765945Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"downstairs","":"downstairs","session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5832026-02-15T00:02:37.194Z{"msg":"Wait for a query_work_queue command to finish before sending IO
5842026-02-15T00:02:37.194Z[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.546807876Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"downstairs","":"downstairs","session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5852026-02-15T00:02:37.194Z{"msg":"[2]R flush_numbers: [5326, 5326, 5326, 5326, 5326]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.546849157Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"downstairs","":"downstairs","session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5862026-02-15T00:02:37.194Z{"msg":"[2]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.546879738Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"downstairs","":"downstairs","session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5872026-02-15T00:02:37.194Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.546921498Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"downstairs","":"downstairs","session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5882026-02-15T00:02:37.194Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.546954039Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"downstairs","":"downstairs","session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5892026-02-15T00:02:37.194Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.54699103Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"downstairs","":"downstairs","session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5902026-02-15T00:02:37.194Z{"msg":"Next flush: 5327","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.547028051Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"downstairs","":"downstairs","session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5912026-02-15T00:02:37.194Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.547059591Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"":"downstairs","":"downstairs","session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5922026-02-15T00:02:37.194Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.547101202Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5932026-02-15T00:02:37.194Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.547132843Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"client":"0","":"downstairs","session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5942026-02-15T00:02:37.194Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.547173964Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"client":"1","":"downstairs","session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5952026-02-15T00:02:37.194Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.547210444Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"client":"2","":"downstairs","session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5962026-02-15T00:02:37.194Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.547238075Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5972026-02-15T00:02:37.194Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.547276946Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5982026-02-15T00:02:37.194Z{"msg":"1cd00e75-c45b-48d8-af30-1ebbb210ab0b is now active with session: 67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.547308526Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
5992026-02-15T00:02:37.194ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
6002026-02-15T00:02:37.194Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.547347037Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262,"session_id":"67500f1f-7aee-4c4b-b6f8-d5ba9b1fa01c"}
6012026-02-15T00:02:37.194Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.547383268Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262}
6022026-02-15T00:02:37.194Z{"msg":"Activated sub_volume 1cd00e75-c45b-48d8-af30-1ebbb210ab0b","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:35.547416759Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262}
6032026-02-15T00:02:37.194ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
6042026-02-15T00:02:37.194ZRead and Verify all blocks (0..100 range:true)
6052026-02-15T00:02:37.280ZWrote out file "/var/tmp/test_up-build/verify_file"
6062026-02-15T00:02:37.280ZVerify test completed
6072026-02-15T00:02:37.284ZWrote out file "/var/tmp/test_up-build/verify_file"
6082026-02-15T00:02:37.284ZCLIENT: Tests done. All submitted work has been ACK'd
6092026-02-15T00:02:37.284Z----------------------------------------------------------------
6102026-02-15T00:02:37.284Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6112026-02-15T00:02:37.284ZStates: Active Active Active
6122026-02-15T00:02:37.284ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6132026-02-15T00:02:37.284Z 1000 Acked Read 50 Done Done Done false
6142026-02-15T00:02:37.284Z STATES DS:0 DS:1 DS:2 TOTAL
6152026-02-15T00:02:37.284Z Sent 0 0 0 0
6162026-02-15T00:02:37.284Z Done 1 1 1 3
6172026-02-15T00:02:37.284Z Skipped 0 0 0 0
6182026-02-15T00:02:37.284Z Error 0 0 0 0
6192026-02-15T00:02:37.284ZLast Flush: None None None
6202026-02-15T00:02:37.284ZDownstairs last five completed:
6212026-02-15T00:02:37.284ZUpstairs last five completed: 1000
6222026-02-15T00:02:37.284Z----------------------------------------------------------------
6232026-02-15T00:02:37.284Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6242026-02-15T00:02:37.284ZStates: Active Active Active
6252026-02-15T00:02:37.284ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6262026-02-15T00:02:37.284Z 1000 Acked Read 50 Done Done Done false
6272026-02-15T00:02:37.284Z STATES DS:0 DS:1 DS:2 TOTAL
6282026-02-15T00:02:37.285Z Sent 0 0 0 0
6292026-02-15T00:02:37.285Z Done 1 1 1 3
6302026-02-15T00:02:37.285Z Skipped 0 0 0 0
6312026-02-15T00:02:37.285Z Error 0 0 0 0
6322026-02-15T00:02:37.285ZLast Flush: None None None
6332026-02-15T00:02:37.285ZDownstairs last five completed:
6342026-02-15T00:02:37.285ZUpstairs last five completed: 1000
6352026-02-15T00:02:37.285ZCLIENT: Up:0 ds:2 act:6
6362026-02-15T00:02:41.282Z----------------------------------------------------------------
6372026-02-15T00:02:41.282Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6382026-02-15T00:02:41.282ZDownstairs last five completed: 1001 1000
6392026-02-15T00:02:41.282ZUpstairs last five completed: 1001 1000
6402026-02-15T00:02:41.282Z----------------------------------------------------------------
6412026-02-15T00:02:41.282Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6422026-02-15T00:02:41.282ZDownstairs last five completed: 1001 1000
6432026-02-15T00:02:41.282ZUpstairs last five completed: 1001 1000
6442026-02-15T00:02:41.286ZCLIENT: Up:0 ds:0 act:6
6452026-02-15T00:02:41.286ZCLIENT: All crucible jobs finished, exiting program
6462026-02-15T00:02:41.286Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:39.637151214Z","hostname":"w-01KHF98GNC38PVJZ1ER1ER3X85","pid":1262}
6472026-02-15T00:02:41.290ZRepair part 2 passed
6482026-02-15T00:02:41.290Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
6492026-02-15T00:02:41.310ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6502026-02-15T00:02:41.310Z 0 000-009 32 32 32 4006 4006 4006 F F F
6512026-02-15T00:02:41.310Z 1 010-019 32 32 32 4006 4006 4006 F F F
6522026-02-15T00:02:41.310Z 2 020-029 32 32 32 4006 4006 4006 F F F
6532026-02-15T00:02:41.310Z 3 030-039 32 32 32 4006 4006 4006 F F F
6542026-02-15T00:02:41.310Z 4 040-049 32 32 32 4006 4006 4006 F F F
6552026-02-15T00:02:41.310ZMax gen: 32, Max flush: 4006
6562026-02-15T00:02:41.315Zdump test passed
6572026-02-15T00:02:41.318Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
6582026-02-15T00:02:41.333Z Extent 1
6592026-02-15T00:02:41.336ZGEN 32 32 32
6602026-02-15T00:02:41.336ZFLUSH_ID 4006 4006 4006
6612026-02-15T00:02:41.336ZDIRTY
6622026-02-15T00:02:41.336Z
6632026-02-15T00:02:41.336ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
6642026-02-15T00:02:41.337Z010 A A A A A A
6652026-02-15T00:02:41.339Z011 A A A A A A
6662026-02-15T00:02:41.339Z012 A A A A A A
6672026-02-15T00:02:41.342Z013 A A A A A A
6682026-02-15T00:02:41.342Z014 A A A A A A
6692026-02-15T00:02:41.345Z015 A A A A A A
6702026-02-15T00:02:41.346Z016 A A A A A A
6712026-02-15T00:02:41.346Z017 A A A A A A
6722026-02-15T00:02:41.348Z018 A A A A A A
6732026-02-15T00:02:41.348Z019 A A A A A A
6742026-02-15T00:02:41.354Zdump extent test passed
6752026-02-15T00:02:41.357Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
6762026-02-15T00:02:41.371ZExtent 2 Block in extent 0 Actual block 20
6772026-02-15T00:02:41.374Z
6782026-02-15T00:02:41.374Z DATA SHA256 VER
6792026-02-15T00:02:41.374Z------ ---------------------------------------------------------------- ---
6802026-02-15T00:02:41.374Z 0 ac3e4999143f15b447173d0c2195bcc13446835d704c7d778de0705ce6af261a A
6812026-02-15T00:02:41.374Z 1 ac3e4999143f15b447173d0c2195bcc13446835d704c7d778de0705ce6af261a A
6822026-02-15T00:02:41.374Z 2 ac3e4999143f15b447173d0c2195bcc13446835d704c7d778de0705ce6af261a A
6832026-02-15T00:02:41.375Z
6842026-02-15T00:02:41.375ZNONCES 0 1 2 DIFF
6852026-02-15T00:02:41.375Z------ ------------------------ ------------------------ ------------------------ -----
6862026-02-15T00:02:41.375Z 0 8cf1265b2f3ee38a07577c50 8cf1265b2f3ee38a07577c50 8cf1265b2f3ee38a07577c50
6872026-02-15T00:02:41.375Z
6882026-02-15T00:02:41.375Z TAGS 0 1 2 DIFF
6892026-02-15T00:02:41.375Z------ -------------------------------- -------------------------------- -------------------------------- -----
6902026-02-15T00:02:41.375Z 0 98630ebf952aace34a4ea569f370de66 98630ebf952aace34a4ea569f370de66 98630ebf952aace34a4ea569f370de66
6912026-02-15T00:02:41.375Z
6922026-02-15T00:02:41.375ZHASHES 0 1 2 DIFF
6932026-02-15T00:02:41.375Z------ ---------------- ---------------- ---------------- -----
6942026-02-15T00:02:41.375Z
6952026-02-15T00:02:41.379Zdump block test passed
6962026-02-15T00:02:41.382ZInitial upstairs tests have completed, stopping all downstairs
6972026-02-15T00:02:46.444ZCreating 7 larger downstairs regions
6982026-02-15T00:02:46.785ZStarting 7 downstairs
6992026-02-15T00:02:51.792Zdsc restarted at PID: 1278
7002026-02-15T00:02:52.203ZNow do the replace-reconcile test
7012026-02-15T00:02:52.262ZUsing 8870 for the replacement port
7022026-02-15T00:06:30.855ZCompleted test: replace-reconcile
7032026-02-15T00:06:30.859ZNow do the replace-before-active test
7042026-02-15T00:12:38.087ZCompleted test: replace-before-active
7052026-02-15T00:12:38.092ZAll tests have completed, stopping all downstairs
7062026-02-15T00:12:38.148Z
7072026-02-15T00:12:38.151ZAll Tests have passed
7082026-02-15T00:12:38.151Z16:11 Test duration
7092026-02-15T00:12:38.151Z
7102026-02-15T00:12:38.151Zreal 16:10.608079929
7112026-02-15T00:12:38.151Zuser 41:05.436792392
7122026-02-15T00:12:38.151Zsys 31.450495526
7132026-02-15T00:12:38.152Ztrap 1.349473285
7142026-02-15T00:12:38.152Ztflt 0.076949204
7152026-02-15T00:12:38.152Zdflt 0.063372681
7162026-02-15T00:12:38.152Zkflt 0.001237509
7172026-02-15T00:12:38.152Zlock 24:05:05.471590859
7182026-02-15T00:12:38.152Zslp 1:14:04.073319587
7192026-02-15T00:12:38.152Zlat 5:12.109685497
7202026-02-15T00:12:38.152Zstop 0.053867897
7212026-02-15T00:12:38.152Z+ echo 'test-up-2region-encrypted ends'
7222026-02-15T00:12:38.152Ztest-up-2region-encrypted ends
7232026-02-15T00:12:43.155Zprocess exited: duration 980924 ms, exit code 0
7242026-02-15T00:12:43.155Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
7252026-02-15T00:13:43.195Zexec warning: : stdout descriptor may be held open by a background process; giving up!
7262026-02-15T00:13:43.195Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
7272026-02-15T00:13:43.207Zfound 17 output files
7282026-02-15T00:13:43.207Zuploading: /tmp/test_up-build/dsc-out.txt (56714 bytes)
7292026-02-15T00:13:44.221Zuploaded: /tmp/test_up-build/dsc-out.txt
7302026-02-15T00:13:44.231Zuploading: /tmp/test_up-build/test_up_out.txt (22589194 bytes)
7312026-02-15T00:13:45.439Zuploaded: /tmp/test_up-build/test_up_out.txt
7322026-02-15T00:13:45.439Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (550950 bytes)
7332026-02-15T00:13:46.461Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
7342026-02-15T00:13:46.464Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (548314 bytes)
7352026-02-15T00:13:47.483Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
7362026-02-15T00:13:47.486Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (979525 bytes)
7372026-02-15T00:13:48.510Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
7382026-02-15T00:13:48.513Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (543013 bytes)
7392026-02-15T00:13:49.532Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
7402026-02-15T00:13:49.532Zuploading: /tmp/test_up-build/dsc/downstairs-8850.txt (0 bytes)
7412026-02-15T00:13:50.539Zuploaded: /tmp/test_up-build/dsc/downstairs-8850.txt
7422026-02-15T00:13:50.542Zuploading: /tmp/test_up-build/dsc/downstairs-8860.txt (44689 bytes)
7432026-02-15T00:13:51.551Zuploaded: /tmp/test_up-build/dsc/downstairs-8860.txt
7442026-02-15T00:13:51.555Zuploading: /tmp/test_up-build/dsc/downstairs-8870.txt (3405956 bytes)
7452026-02-15T00:13:52.596Zuploaded: /tmp/test_up-build/dsc/downstairs-8870.txt
7462026-02-15T00:13:52.599Zuploading: /tmp/debug/df.txt (1270 bytes)
7472026-02-15T00:13:53.608Zuploaded: /tmp/debug/df.txt
7482026-02-15T00:13:53.611Zuploading: /tmp/debug/dtrace.txt (1431398 bytes)
7492026-02-15T00:13:54.637Zuploaded: /tmp/debug/dtrace.txt
7502026-02-15T00:13:54.641Zuploading: /tmp/debug/iostat.txt (312086 bytes)
7512026-02-15T00:13:54.651Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 312086 -> 315698
7522026-02-15T00:13:55.657Zuploaded: /tmp/debug/iostat.txt
7532026-02-15T00:13:55.661Zuploading: /tmp/debug/mpstat.txt (756314 bytes)
7542026-02-15T00:13:55.677Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 756314 -> 765804
7552026-02-15T00:13:56.683Zuploaded: /tmp/debug/mpstat.txt
7562026-02-15T00:13:56.687Zuploading: /tmp/debug/paging.txt (136608 bytes)
7572026-02-15T00:13:56.694Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 136608 -> 138390
7582026-02-15T00:13:57.702Zuploaded: /tmp/debug/paging.txt
7592026-02-15T00:13:57.705Zuploading: /tmp/debug/prstat.txt (1497449 bytes)
7602026-02-15T00:13:57.727Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 1497449 -> 1515544
7612026-02-15T00:13:58.734Zuploaded: /tmp/debug/prstat.txt
7622026-02-15T00:13:58.737Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
7632026-02-15T00:13:59.744Zuploaded: /tmp/debug/psrinfo.txt
7642026-02-15T00:13:59.747Zuploading: /tmp/debug/upstairs-info.txt (235858 bytes)
7652026-02-15T00:14:00.761Zuploaded: /tmp/debug/upstairs-info.txt