01KS3Q1476THY2E1NNTWPTVDT5: test-up-2region-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01KS3Q1FSGV0ZB25QVJRFD3BKG

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12026-05-20T22:24:10.829Zjob dependencies complete; ready to run (waiting for 13 m 38 s)
22026-05-20T22:25:32.283Zjob assigned to worker 01KS3QV8DJR41ZPDFCSQ6C99N5 [factory aws, i-0db831b6d05ebe6ab] (queued for 1 m 21 s)
32026-05-20T22:25:39.769Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42026-05-20T22:25:42.198Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52026-05-20T22:25:42.252Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62026-05-20T22:25:45.585Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72026-05-20T22:25:45.668Zdownloading input: /input/build/work/bins/crutest.gz
82026-05-20T22:25:48.445Zdownloaded input: /input/build/work/bins/crutest.gz
92026-05-20T22:25:48.449Zdownloading input: /input/build/work/bins/dsc.gz
102026-05-20T22:25:49.366Zdownloaded input: /input/build/work/bins/dsc.gz
112026-05-20T22:25:49.370Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122026-05-20T22:25:49.395Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132026-05-20T22:25:49.399Zdownloading input: /input/build/work/scripts/test_ds.sh
142026-05-20T22:25:49.444Zdownloaded input: /input/build/work/scripts/test_ds.sh
152026-05-20T22:25:49.459Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162026-05-20T22:25:49.482Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172026-05-20T22:25:49.483Zdownloading input: /input/build/work/scripts/test_repair.sh
182026-05-20T22:25:49.521Zdownloaded input: /input/build/work/scripts/test_repair.sh
192026-05-20T22:25:49.526Zdownloading input: /input/build/work/scripts/test_replay.sh
202026-05-20T22:25:49.548Zdownloaded input: /input/build/work/scripts/test_replay.sh
212026-05-20T22:25:49.548Zdownloading input: /input/build/work/scripts/test_up.sh
222026-05-20T22:25:49.573Zdownloaded input: /input/build/work/scripts/test_up.sh
232026-05-20T22:25:49.577Zdownloading input: /input/build/work/scripts/upstairs_info.d
242026-05-20T22:25:49.605Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252026-05-20T22:25:49.613Zdownloading input: /input/build/tmp/cargo-test-out.log
262026-05-20T22:25:50.282Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272026-05-20T22:25:50.282Zstarting task 0: "setup"
282026-05-20T22:25:50.289Z++ uname -s
292026-05-20T22:25:50.296Z+ kern=SunOS
302026-05-20T22:25:50.296Z+ build_user=build
312026-05-20T22:25:50.296Z+ build_uid=12345
322026-05-20T22:25:50.296Z+ work_dir=/work
332026-05-20T22:25:50.296Z+ input_dir=/input
342026-05-20T22:25:50.296Z+ [[ 0 == 12345 ]]
352026-05-20T22:25:50.296Z+ case "$kern" in
362026-05-20T22:25:50.296Z+ groupadd -g 12345 build
372026-05-20T22:25:50.296Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382026-05-20T22:25:52.297Z+ zfs create -o mountpoint=/work rpool/work
392026-05-20T22:25:52.564Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402026-05-20T22:25:52.573Z+ home_fs=zfs
412026-05-20T22:25:52.573Z+ [[ zfs == autofs ]]
422026-05-20T22:25:52.573Z+ mkdir -p /home/build
432026-05-20T22:25:52.573Z+ chown build:build /home/build /work
442026-05-20T22:25:54.570Z+ chmod 0700 /home/build /work
452026-05-20T22:25:54.580Zprocess exited: duration 4283 ms, exit code 0
 
462026-05-20T22:25:54.592Zstarting task 1: "authentication"
472026-05-20T22:25:54.611Zprocess exited: duration 19 ms, exit code 0
 
482026-05-20T22:25:54.619Zstarting task 2: "build"
492026-05-20T22:25:54.623Z+ banner cores
502026-05-20T22:25:54.628Z
512026-05-20T22:25:54.628Z #### #### ##### ###### ####
522026-05-20T22:25:54.628Z # # # # # # # #
532026-05-20T22:25:54.628Z # # # # # ##### ####
542026-05-20T22:25:54.628Z # # # ##### # #
552026-05-20T22:25:54.628Z # # # # # # # # #
562026-05-20T22:25:54.628Z #### #### # # ###### ####
572026-05-20T22:25:54.628Z
582026-05-20T22:25:54.628Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
592026-05-20T22:25:54.632Z+ banner unpack
602026-05-20T22:25:54.635Z
612026-05-20T22:25:54.635Z # # # # ##### ## #### # #
622026-05-20T22:25:54.635Z # # ## # # # # # # # # #
632026-05-20T22:25:54.635Z # # # # # # # # # # ####
642026-05-20T22:25:54.635Z # # # # # ##### ###### # # #
652026-05-20T22:25:54.636Z # # # ## # # # # # # #
662026-05-20T22:25:54.636Z #### # # # # # #### # #
672026-05-20T22:25:54.636Z
682026-05-20T22:25:54.636Z+ mkdir -p /var/tmp/bins
692026-05-20T22:25:54.636Z+ for t in "$input/bins/"*.gz
702026-05-20T22:25:54.636Z++ basename /input/build/work/bins/crucible-downstairs.gz
712026-05-20T22:25:54.636Z+ b=crucible-downstairs.gz
722026-05-20T22:25:54.636Z+ b=crucible-downstairs
732026-05-20T22:25:54.636Z+ gunzip
742026-05-20T22:25:57.327Z+ chmod +x /var/tmp/bins/crucible-downstairs
752026-05-20T22:25:57.331Z+ for t in "$input/bins/"*.gz
762026-05-20T22:25:57.331Z++ basename /input/build/work/bins/crucible-hammer.gz
772026-05-20T22:25:57.334Z+ b=crucible-hammer.gz
782026-05-20T22:25:57.334Z+ b=crucible-hammer
792026-05-20T22:25:57.334Z+ gunzip
802026-05-20T22:25:59.862Z+ chmod +x /var/tmp/bins/crucible-hammer
812026-05-20T22:25:59.865Z+ for t in "$input/bins/"*.gz
822026-05-20T22:25:59.865Z++ basename /input/build/work/bins/crutest.gz
832026-05-20T22:25:59.868Z+ b=crutest.gz
842026-05-20T22:25:59.868Z+ b=crutest
852026-05-20T22:25:59.868Z+ gunzip
862026-05-20T22:26:02.510Z+ chmod +x /var/tmp/bins/crutest
872026-05-20T22:26:02.517Z+ for t in "$input/bins/"*.gz
882026-05-20T22:26:02.517Z++ basename /input/build/work/bins/dsc.gz
892026-05-20T22:26:02.517Z+ b=dsc.gz
902026-05-20T22:26:02.517Z+ b=dsc
912026-05-20T22:26:02.517Z+ gunzip
922026-05-20T22:26:03.778Z+ chmod +x /var/tmp/bins/dsc
932026-05-20T22:26:03.784Z+ export BINDIR=/var/tmp/bins
942026-05-20T22:26:03.784Z+ BINDIR=/var/tmp/bins
952026-05-20T22:26:03.784Z+ jobpid=1091
962026-05-20T22:26:03.784Z+ echo 'Setup debug logging'
972026-05-20T22:26:03.784Z+ mkdir /tmp/debug
982026-05-20T22:26:03.784ZSetup debug logging
992026-05-20T22:26:03.785Z+ sleep 3600
1002026-05-20T22:26:03.785Z+ psrinfo -v
1012026-05-20T22:26:03.785Z+ df -h
1022026-05-20T22:26:03.785Z+ prstat -d d -mLc 1
1032026-05-20T22:26:03.785Z+ iostat -T d -xn 1
1042026-05-20T22:26:03.785Z+ mpstat -T d 1
1052026-05-20T22:26:03.785Z+ vmstat -T d -p 1
1062026-05-20T22:26:03.785Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1072026-05-20T22:26:03.785Z+ banner test_up_2r_encrypted
1082026-05-20T22:26:03.785Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1092026-05-20T22:26:03.785Z #####
1102026-05-20T22:26:03.785Z ##### ###### #### ##### # # ##### # # #####
1112026-05-20T22:26:03.785Z # # # # # # # # # # #
1122026-05-20T22:26:03.785Z # ##### #### # # # # # ##### # #
1132026-05-20T22:26:03.785Z # # # # # # ##### # #####
1142026-05-20T22:26:03.785Z # # # # # # # # # # #
1152026-05-20T22:26:03.785Z # ###### #### # ####### #### # ####### ####### # #
1162026-05-20T22:26:03.785Z
1172026-05-20T22:26:03.786Z+ ptime -m bash /input/build/work/scripts/test_up.sh -r 2 -N encrypted
1182026-05-20T22:26:03.792Z/input/build/work
1192026-05-20T22:26:03.795ZUsing 2 region sets
1202026-05-20T22:26:03.795ZTurn off color for downstairs dump
1212026-05-20T22:26:03.821ZUpstairs using key: 99GuRqRlXDbJ63iKGTcYBOwIfPYoIMyQyQAw+iYS0AM=
1222026-05-20T22:26:03.828Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1232026-05-20T22:26:03.828ZCreating 6 downstairs regions
1242026-05-20T22:26:04.050ZStarting 6 downstairs
1252026-05-20T22:26:04.055Zdsc started at PID: 1139
1262026-05-20T22:26:09.144ZDisable automatic restart on all downstairs
1272026-05-20T22:26:09.156Z
1282026-05-20T22:26:09.156ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1292026-05-20T22:26:09.164ZRunning test: span
1302026-05-20T22:26:13.255ZCompleted test: span
1312026-05-20T22:26:18.259ZRunning test: big
1322026-05-20T22:26:22.570ZCompleted test: big
1332026-05-20T22:26:27.577ZRunning test: dep
1342026-05-20T22:26:47.223ZCompleted test: dep
1352026-05-20T22:26:52.230ZRunning test: balloon
1362026-05-20T22:29:19.909ZCompleted test: balloon
1372026-05-20T22:29:24.915ZRunning test: deactivate
1382026-05-20T22:29:29.247ZCompleted test: deactivate
1392026-05-20T22:29:34.254ZRunning hammer
1402026-05-20T22:29:41.497ZRun repair tests
1412026-05-20T22:29:41.501Z/var/tmp/bins/crutest fill -g 31 -q --key 99GuRqRlXDbJ63iKGTcYBOwIfPYoIMyQyQAw+iYS0AM= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1422026-05-20T22:29:41.517Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.563379174Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"task":"crutest"}
1432026-05-20T22:29:41.523Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.563702593Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"task":"crutest"}
1442026-05-20T22:29:41.524Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.564502847Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"task":"crutest"}
1452026-05-20T22:29:41.524Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.564756692Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182}
1462026-05-20T22:29:41.524Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"8a081a6676ca7e9272269457d75f77537f9f2512\",\n git_commit_timestamp: \"2026-05-20T22:09:56.000000000Z\",\n git_branch: \"sunshowers/spr/meta-update-dropshot-api-manager-to-072\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.564799988Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182}
1472026-05-20T22:29:41.524Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.564809837Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182}
1482026-05-20T22:29:41.524Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.564822476Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182}
1492026-05-20T22:29:41.524Z{"msg":"Crucible 081e89a8-612d-4033-9496-71cd4fca53c1 has session id: 1cb406b2-b0f7-41d0-96ee-7cd24b5038b1","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.56519559Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1502026-05-20T22:29:41.524Z{"msg":"Upstairs opts: Upstairs UUID: 081e89a8-612d-4033-9496-71cd4fca53c1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.565230797Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1512026-05-20T22:29:41.524Z{"msg":"Crucible stats registered with UUID: 081e89a8-612d-4033-9496-71cd4fca53c1","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.565242716Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1522026-05-20T22:29:41.524Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-20T22:29:40.565252915Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1532026-05-20T22:29:41.524Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.565393181Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1542026-05-20T22:29:41.524Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.565442847Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1552026-05-20T22:29:41.524Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.565469384Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1562026-05-20T22:29:41.524Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.566310074Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"task":"crutest"}
1572026-05-20T22:29:41.524Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.566369718Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182}
1582026-05-20T22:29:41.524Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"8a081a6676ca7e9272269457d75f77537f9f2512\",\n git_commit_timestamp: \"2026-05-20T22:09:56.000000000Z\",\n git_branch: \"sunshowers/spr/meta-update-dropshot-api-manager-to-072\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.566391046Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182}
1592026-05-20T22:29:41.524Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.566400205Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182}
1602026-05-20T22:29:41.524Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.566412444Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182}
1612026-05-20T22:29:41.524Z{"msg":"Crucible 081e89a8-612d-4033-9496-71cd4fca53c1 has session id: b735f7b5-b4b1-4a66-b9d1-5b83d8211666","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.566615934Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
1622026-05-20T22:29:41.524Z{"msg":"Upstairs opts: Upstairs UUID: 081e89a8-612d-4033-9496-71cd4fca53c1, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.566637322Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
1632026-05-20T22:29:41.524Z{"msg":"Crucible stats registered with UUID: 081e89a8-612d-4033-9496-71cd4fca53c1","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.566647751Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
1642026-05-20T22:29:41.524Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-20T22:29:40.56665734Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
1652026-05-20T22:29:41.524Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.566666399Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
1662026-05-20T22:29:41.524Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.566695607Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
1672026-05-20T22:29:41.524Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.566711055Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
1682026-05-20T22:29:41.525Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.566797727Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"task":"crutest"}
1692026-05-20T22:29:41.525Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.566811096Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182}
1702026-05-20T22:29:41.525Z{"msg":"081e89a8-612d-4033-9496-71cd4fca53c1 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.566850502Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1712026-05-20T22:29:41.525Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.566864051Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"client":"0","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1722026-05-20T22:29:41.525Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.56687454Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"client":"1","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1732026-05-20T22:29:41.525Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.566888748Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"client":"2","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1742026-05-20T22:29:41.525Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.566898867Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1752026-05-20T22:29:41.525Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.566913436Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1762026-05-20T22:29:41.525Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.566923325Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1772026-05-20T22:29:41.525Z{"msg":"ds_connection connected from Ok(127.0.0.1:56714)","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.566964991Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1782026-05-20T22:29:41.525Z{"msg":"ds_connection connected from Ok(127.0.0.1:49227)","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.567059252Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1792026-05-20T22:29:41.525Z{"msg":"ds_connection connected from Ok(127.0.0.1:51219)","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.567087229Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1802026-05-20T22:29:41.555Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.567623638Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"client":"0","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1812026-05-20T22:29:41.555Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.567652985Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"client":"1","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1822026-05-20T22:29:41.555Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.567665234Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"client":"2","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1832026-05-20T22:29:41.555Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.56781205Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1842026-05-20T22:29:41.556Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.567852756Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1852026-05-20T22:29:41.556Z{"msg":"[0]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.567889872Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"downstairs","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1862026-05-20T22:29:41.556Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.567900821Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"downstairs","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1872026-05-20T22:29:41.556Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.56791062Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"downstairs","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1882026-05-20T22:29:41.556Z{"msg":"[1]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.567921199Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"downstairs","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1892026-05-20T22:29:41.556Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.567931328Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"downstairs","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1902026-05-20T22:29:41.556Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.567941547Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"downstairs","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1912026-05-20T22:29:41.556Z{"msg":"[2]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.567951716Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"downstairs","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1922026-05-20T22:29:41.556Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.567962005Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"downstairs","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1932026-05-20T22:29:41.556Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.567972204Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"downstairs","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1942026-05-20T22:29:41.556Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.567982014Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"downstairs","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1952026-05-20T22:29:41.556Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.567991833Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"downstairs","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1962026-05-20T22:29:41.556Z{"msg":"Next flush: 3999","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568001652Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"downstairs","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1972026-05-20T22:29:41.556Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568012111Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"downstairs","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1982026-05-20T22:29:41.556Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.56802262Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
1992026-05-20T22:29:41.556Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568032269Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"client":"0","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
2002026-05-20T22:29:41.556Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568042188Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"client":"1","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
2012026-05-20T22:29:41.556Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568052317Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"client":"2","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
2022026-05-20T22:29:41.556Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568062116Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
2032026-05-20T22:29:41.556Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568072185Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
2042026-05-20T22:29:41.556Z{"msg":"081e89a8-612d-4033-9496-71cd4fca53c1 is now active with session: 1cb406b2-b0f7-41d0-96ee-7cd24b5038b1","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568083334Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
2052026-05-20T22:29:41.556Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568092473Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
2062026-05-20T22:29:41.556Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568102372Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182}
2072026-05-20T22:29:41.556Z{"msg":"Activated sub_volume 081e89a8-612d-4033-9496-71cd4fca53c1","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568110561Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182}
2082026-05-20T22:29:41.556Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.56811929Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182}
2092026-05-20T22:29:41.556Z{"msg":"081e89a8-612d-4033-9496-71cd4fca53c1 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.56812771Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2102026-05-20T22:29:41.556Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568136519Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"client":"0","":"downstairs","session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2112026-05-20T22:29:41.556Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568150487Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"client":"1","":"downstairs","session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2122026-05-20T22:29:41.556Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568164076Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"client":"2","":"downstairs","session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2132026-05-20T22:29:41.556Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568173925Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2142026-05-20T22:29:41.556Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568187914Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2152026-05-20T22:29:41.556Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568197913Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2162026-05-20T22:29:41.556Z{"msg":"ds_connection connected from Ok(127.0.0.1:51573)","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568208472Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2172026-05-20T22:29:41.556Z{"msg":"ds_connection connected from Ok(127.0.0.1:65501)","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568219001Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2182026-05-20T22:29:41.556Z{"msg":"ds_connection connected from Ok(127.0.0.1:56195)","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568271996Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2192026-05-20T22:29:41.557Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568583776Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"client":"0","":"downstairs","session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2202026-05-20T22:29:41.557Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568685626Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"client":"1","":"downstairs","session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2212026-05-20T22:29:41.557Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568744441Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2222026-05-20T22:29:41.557Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568755859Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"client":"2","":"downstairs","session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2232026-05-20T22:29:41.557Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568838402Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2242026-05-20T22:29:41.557Z{"msg":"[0]R flush_numbers: [5312, 5310, 5316, 5315, 5310]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568889177Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"downstairs","":"downstairs","session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2252026-05-20T22:29:41.557Z{"msg":"[0]R generation: [13, 12, 14, 14, 12]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568900136Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"downstairs","":"downstairs","session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2262026-05-20T22:29:41.557Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568910505Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"downstairs","":"downstairs","session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2272026-05-20T22:29:41.557Z{"msg":"[1]R flush_numbers: [5312, 5310, 5316, 5315, 5310]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568920924Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"downstairs","":"downstairs","session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2282026-05-20T22:29:41.557Z{"msg":"[1]R generation: [13, 12, 14, 14, 12]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568935672Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"downstairs","":"downstairs","session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2292026-05-20T22:29:41.557Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568945861Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"downstairs","":"downstairs","session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2302026-05-20T22:29:41.557Z{"msg":"[2]R flush_numbers: [5312, 5310, 5316, 5315, 5310]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.56895661Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"downstairs","":"downstairs","session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2312026-05-20T22:29:41.557Z{"msg":"[2]R generation: [13, 12, 14, 14, 12]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568966889Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"downstairs","":"downstairs","session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2322026-05-20T22:29:41.557Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568977168Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"downstairs","":"downstairs","session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2332026-05-20T22:29:41.557Z{"msg":"Max found gen is 15","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568987017Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"downstairs","":"downstairs","session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2342026-05-20T22:29:41.557Z{"msg":"Generation requested: 31 >= found:15","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.568997496Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"downstairs","":"downstairs","session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2352026-05-20T22:29:41.557Z{"msg":"Next flush: 5317","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.569006775Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"downstairs","":"downstairs","session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2362026-05-20T22:29:41.557Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.569016484Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"downstairs","":"downstairs","session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2372026-05-20T22:29:41.557Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.569026843Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2382026-05-20T22:29:41.557Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.569036223Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"client":"0","":"downstairs","session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2392026-05-20T22:29:41.557Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.569046262Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"client":"1","":"downstairs","session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2402026-05-20T22:29:41.557Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.569056361Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"client":"2","":"downstairs","session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2412026-05-20T22:29:41.557Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.5690666Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2422026-05-20T22:29:41.557Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.569076799Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2432026-05-20T22:29:41.557Z{"msg":"081e89a8-612d-4033-9496-71cd4fca53c1 is now active with session: b735f7b5-b4b1-4a66-b9d1-5b83d8211666","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.569087518Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2442026-05-20T22:29:41.557Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.569096647Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"session_id":"b735f7b5-b4b1-4a66-b9d1-5b83d8211666"}
2452026-05-20T22:29:41.557Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.569106476Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182}
2462026-05-20T22:29:41.557Z{"msg":"Activated sub_volume 081e89a8-612d-4033-9496-71cd4fca53c1","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.569115185Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182}
2472026-05-20T22:29:41.557Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.569124044Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"task":"crutest"}
2482026-05-20T22:29:41.557ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
2492026-05-20T22:29:41.557Z{"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.569250742Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"task":"crutest"}
2502026-05-20T22:29:41.557ZRead and Verify all blocks (0..100 range:false)
2512026-05-20T22:29:41.596Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.641810715Z----------------------------------------------------------------
2522026-05-20T22:29:41.596Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182 Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2532026-05-20T22:29:41.596Z,States:"task":" Activecrutest" Active Active
2542026-05-20T22:29:41.596Z}JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2552026-05-20T22:29:41.596Z
2562026-05-20T22:29:41.596Z 1002 Acked Read 50{ Done" Donemsg Done" false:"
2572026-05-20T22:29:41.600ZCLIENT: Tests done. All submitted work has been ACK'd","v":0,"name":"crucible" STATES DS:0 DS:1 DS:2 TOTAL
2582026-05-20T22:29:41.600Z," Sent level": 0 30 0 0 0
2592026-05-20T22:29:41.600Z Done 1 1 1 3
2602026-05-20T22:29:41.600Z Skipped 0 0 0 0
2612026-05-20T22:29:41.600Z," Error time 0 " 0 : 0 " 0
2622026-05-20T22:29:41.600Z2026-05-20T22:29:40.641847761Z","Last Flush: Some(JobId(1001)) hostnameSome(JobId(1001)) ":Some(JobId(1001)) "
2632026-05-20T22:29:41.600Zw-01KS3QV8DJR41ZPDFCSQ6C99N5Downstairs last five completed:","pid":1182,"task": 1001" 1000
2642026-05-20T22:29:41.600Zcrutest"}
2652026-05-20T22:29:41.601ZUpstairs last five completed: 1002 1001 1000
2662026-05-20T22:29:41.601Z----------------------------------------------------------------
2672026-05-20T22:29:41.601Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2682026-05-20T22:29:41.601ZStates: Active Active Active
2692026-05-20T22:29:41.601ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2702026-05-20T22:29:41.601Z 1002 Acked Read 50 Done Done Done false
2712026-05-20T22:29:41.601Z STATES DS:0 DS:1 DS:2 TOTAL
2722026-05-20T22:29:41.601Z Sent 0 0 0 0
2732026-05-20T22:29:41.601Z Done 1 1 1 3
2742026-05-20T22:29:41.601Z Skipped 0 0 0 0
2752026-05-20T22:29:41.601Z Error 0 0 0 0
2762026-05-20T22:29:41.601ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2772026-05-20T22:29:41.601ZDownstairs last five completed: 1001 1000
2782026-05-20T22:29:41.601ZUpstairs last five completed: 1002 1001 1000
2792026-05-20T22:29:41.601Z{"msg":"CLIENT: Up:0 ds:2 act:6","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:40.641919034Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"task":"crutest"}
2802026-05-20T22:29:45.599Z----------------------------------------------------------------
2812026-05-20T22:29:45.599Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2822026-05-20T22:29:45.599ZDownstairs last five completed: 1003 1002 1001 1000
2832026-05-20T22:29:45.599ZUpstairs last five completed: 1003 1002 1001 1000
2842026-05-20T22:29:45.602Z----------------------------------------------------------------
2852026-05-20T22:29:45.602Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2862026-05-20T22:29:45.603ZDownstairs last five completed: 1003 1002 1001 1000
2872026-05-20T22:29:45.603ZUpstairs last five completed: 1003 1002 1001 1000
2882026-05-20T22:29:45.603Z{"msg":"CLIENT: Up:0 ds:0 act:6","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.643772577Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"task":"crutest"}
2892026-05-20T22:29:45.603Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.64385081Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"task":"crutest"}
2902026-05-20T22:29:45.603Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-20T22:29:44.643873817Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182}
2912026-05-20T22:29:45.603Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-20T22:29:44.643891056Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182}
2922026-05-20T22:29:45.603Z{"msg":"client request queue closed unexpectedly; is the program exiting?","v":0,"name":"crucible","level":40,"time":"2026-05-20T22:29:44.644037832Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
2932026-05-20T22:29:45.603Z{"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2026-05-20T22:29:44.644065139Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
2942026-05-20T22:29:45.603Z{"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2026-05-20T22:29:44.644089697Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
2952026-05-20T22:29:45.603Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.644112075Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
2962026-05-20T22:29:45.603Z{"msg":"client request queue closed unexpectedly; is the program exiting?","v":0,"name":"crucible","level":40,"time":"2026-05-20T22:29:44.644136112Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
2972026-05-20T22:29:45.603Z{"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2026-05-20T22:29:44.64415967Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
2982026-05-20T22:29:45.603Z{"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2026-05-20T22:29:44.644182818Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
2992026-05-20T22:29:45.603Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.644205236Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"1cb406b2-b0f7-41d0-96ee-7cd24b5038b1"}
3002026-05-20T22:29:45.607ZRepair setup passed
3012026-05-20T22:29:45.610ZCopy the region for /var/tmp/test_up-build/8830
3022026-05-20T22:29:45.610Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
3032026-05-20T22:29:45.613Z/var/tmp/bins/crutest fill -g 32 -q --key 99GuRqRlXDbJ63iKGTcYBOwIfPYoIMyQyQAw+iYS0AM= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
3042026-05-20T22:29:45.631Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.67663693Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"task":"crutest"}
3052026-05-20T22:29:45.634Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.677001586Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"task":"crutest"}
3062026-05-20T22:29:45.634Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.678142286Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"task":"crutest"}
3072026-05-20T22:29:45.634Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.678400032Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188}
3082026-05-20T22:29:45.634Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"8a081a6676ca7e9272269457d75f77537f9f2512\",\n git_commit_timestamp: \"2026-05-20T22:09:56.000000000Z\",\n git_branch: \"sunshowers/spr/meta-update-dropshot-api-manager-to-072\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.678440838Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188}
3092026-05-20T22:29:45.634Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.678454306Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188}
3102026-05-20T22:29:45.634Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.678466975Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188}
3112026-05-20T22:29:45.634Z{"msg":"Crucible 07d4e7f7-7406-48c7-a631-543d3b23baf9 has session id: d0423579-e81b-4451-a1af-1f938075dd77","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.678913273Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3122026-05-20T22:29:45.634Z{"msg":"Upstairs opts: Upstairs UUID: 07d4e7f7-7406-48c7-a631-543d3b23baf9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.67893941Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3132026-05-20T22:29:45.634Z{"msg":"Crucible stats registered with UUID: 07d4e7f7-7406-48c7-a631-543d3b23baf9","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.678949909Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3142026-05-20T22:29:45.634Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-20T22:29:44.678958998Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3152026-05-20T22:29:45.635Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.679127272Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3162026-05-20T22:29:45.635Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.679179207Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3172026-05-20T22:29:45.635Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.679193296Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3182026-05-20T22:29:45.639Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.680427508Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"task":"crutest"}
3192026-05-20T22:29:45.639Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.68050633Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188}
3202026-05-20T22:29:45.639Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"8a081a6676ca7e9272269457d75f77537f9f2512\",\n git_commit_timestamp: \"2026-05-20T22:09:56.000000000Z\",\n git_branch: \"sunshowers/spr/meta-update-dropshot-api-manager-to-072\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.680535557Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188}
3212026-05-20T22:29:45.639Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.680550326Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188}
3222026-05-20T22:29:45.639Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.680564434Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188}
3232026-05-20T22:29:45.639Z{"msg":"Crucible 07d4e7f7-7406-48c7-a631-543d3b23baf9 has session id: be31a740-d5a1-4f0e-b86a-a8ff0db414d3","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.680790783Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3242026-05-20T22:29:45.639Z{"msg":"Upstairs opts: Upstairs UUID: 07d4e7f7-7406-48c7-a631-543d3b23baf9, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.68081731Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3252026-05-20T22:29:45.639Z{"msg":"Crucible stats registered with UUID: 07d4e7f7-7406-48c7-a631-543d3b23baf9","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.680828789Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3262026-05-20T22:29:45.640Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-20T22:29:44.680840938Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3272026-05-20T22:29:45.640Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.680856177Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3282026-05-20T22:29:45.640Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.680867975Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3292026-05-20T22:29:45.640Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.680884224Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3302026-05-20T22:29:45.640Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.681021531Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"task":"crutest"}
3312026-05-20T22:29:45.640Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.68103291Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188}
3322026-05-20T22:29:45.640Z{"msg":"07d4e7f7-7406-48c7-a631-543d3b23baf9 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.681052648Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3332026-05-20T22:29:45.640Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.681067976Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"client":"0","":"downstairs","session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3342026-05-20T22:29:45.640Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.681079245Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"client":"1","":"downstairs","session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3352026-05-20T22:29:45.640Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.681095334Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"client":"2","":"downstairs","session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3362026-05-20T22:29:45.640Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.681107312Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3372026-05-20T22:29:45.640Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.68113045Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3382026-05-20T22:29:45.640Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.681147109Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3392026-05-20T22:29:45.640Z{"msg":"ds_connection connected from Ok(127.0.0.1:60478)","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.681164117Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3402026-05-20T22:29:45.640Z{"msg":"ds_connection connected from Ok(127.0.0.1:37627)","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.681176996Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3412026-05-20T22:29:45.640Z{"msg":"ds_connection connected from Ok(127.0.0.1:37655)","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.681192814Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3422026-05-20T22:29:45.640Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.681923614Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"client":"1","":"downstairs","session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3432026-05-20T22:29:45.640Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.681938773Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"client":"2","":"downstairs","session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3442026-05-20T22:29:45.640Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.681959181Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"client":"0","":"downstairs","session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3452026-05-20T22:29:45.640Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682145643Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3462026-05-20T22:29:45.640Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682159722Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3472026-05-20T22:29:45.640Z{"msg":"[0]R flush_numbers: [3999, 3999, 3999, 3999, 3999]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682200048Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"downstairs","":"downstairs","session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3482026-05-20T22:29:45.640Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682214656Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"downstairs","":"downstairs","session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3492026-05-20T22:29:45.640Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682224546Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"downstairs","":"downstairs","session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3502026-05-20T22:29:45.640Z{"msg":"[1]R flush_numbers: [3999, 3999, 3999, 3999, 3999]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682239144Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"downstairs","":"downstairs","session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3512026-05-20T22:29:45.640Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682249083Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"downstairs","":"downstairs","session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3522026-05-20T22:29:45.640Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682259052Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"downstairs","":"downstairs","session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3532026-05-20T22:29:45.640Z{"msg":"[2]R flush_numbers: [3999, 3999, 3999, 3999, 3999]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682269481Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"downstairs","":"downstairs","session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3542026-05-20T22:29:45.640Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.6822835Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"downstairs","":"downstairs","session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3552026-05-20T22:29:45.640Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682293759Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"downstairs","":"downstairs","session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3562026-05-20T22:29:45.640Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682303558Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"downstairs","":"downstairs","session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3572026-05-20T22:29:45.640Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682313467Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"downstairs","":"downstairs","session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3582026-05-20T22:29:45.640Z{"msg":"Next flush: 4000","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682326966Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"downstairs","":"downstairs","session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3592026-05-20T22:29:45.640Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682336635Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"downstairs","":"downstairs","session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3602026-05-20T22:29:45.640Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682350723Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3612026-05-20T22:29:45.640Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682360013Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"client":"0","":"downstairs","session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3622026-05-20T22:29:45.640Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682373761Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"client":"1","":"downstairs","session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3632026-05-20T22:29:45.640Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.68238382Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"client":"2","":"downstairs","session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3642026-05-20T22:29:45.641Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682398039Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3652026-05-20T22:29:45.641Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682407598Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3662026-05-20T22:29:45.641Z{"msg":"07d4e7f7-7406-48c7-a631-543d3b23baf9 is now active with session: d0423579-e81b-4451-a1af-1f938075dd77","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682418317Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3672026-05-20T22:29:45.641Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682431076Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"session_id":"d0423579-e81b-4451-a1af-1f938075dd77"}
3682026-05-20T22:29:45.641Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682484121Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188}
3692026-05-20T22:29:45.641Z{"msg":"Activated sub_volume 07d4e7f7-7406-48c7-a631-543d3b23baf9","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682538295Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188}
3702026-05-20T22:29:45.641Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682558234Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188}
3712026-05-20T22:29:45.641Z{"msg":"07d4e7f7-7406-48c7-a631-543d3b23baf9 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682576452Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3722026-05-20T22:29:45.641Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682586111Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"client":"0","":"downstairs","session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3732026-05-20T22:29:45.641Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.68260025Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"client":"1","":"downstairs","session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3742026-05-20T22:29:45.641Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682610539Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"client":"2","":"downstairs","session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3752026-05-20T22:29:45.641Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682624497Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3762026-05-20T22:29:45.641Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682634366Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3772026-05-20T22:29:45.641Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682648385Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3782026-05-20T22:29:45.641Z{"msg":"ds_connection connected from Ok(127.0.0.1:51018)","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682658664Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3792026-05-20T22:29:45.641Z{"msg":"ds_connection connected from Ok(127.0.0.1:56610)","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682672963Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3802026-05-20T22:29:45.641Z{"msg":"ds_connection connected from Ok(127.0.0.1:52447)","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682683272Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3812026-05-20T22:29:45.641Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.68280224Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"client":"0","":"downstairs","session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3822026-05-20T22:29:45.641Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682864294Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"client":"1","":"downstairs","session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3832026-05-20T22:29:45.641Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.682897991Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"client":"2","":"downstairs","session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3842026-05-20T22:29:45.641Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.683018889Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3852026-05-20T22:29:45.641Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.683032968Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3862026-05-20T22:29:45.641Z{"msg":"[0]R flush_numbers: [5317, 5317, 5317, 5317, 5317]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.683042537Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"downstairs","":"downstairs","session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3872026-05-20T22:29:45.641Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.683056776Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"downstairs","":"downstairs","session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3882026-05-20T22:29:45.641Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.683067335Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"downstairs","":"downstairs","session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3892026-05-20T22:29:45.641Z{"msg":"[1]R flush_numbers: [5317, 5317, 5317, 5317, 5317]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.683077244Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"downstairs","":"downstairs","session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3902026-05-20T22:29:45.641Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.683091293Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"downstairs","":"downstairs","session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3912026-05-20T22:29:45.641Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.683101272Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"downstairs","":"downstairs","session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3922026-05-20T22:29:45.641Z{"msg":"[2]R flush_numbers: [5317, 5317, 5317, 5317, 5317]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.68311584Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"downstairs","":"downstairs","session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3932026-05-20T22:29:45.641Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.683125929Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"downstairs","":"downstairs","session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3942026-05-20T22:29:45.641Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.683140018Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"downstairs","":"downstairs","session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3952026-05-20T22:29:45.641Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.683149387Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"downstairs","":"downstairs","session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3962026-05-20T22:29:45.641Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.683163206Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"downstairs","":"downstairs","session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3972026-05-20T22:29:45.641Z{"msg":"Next flush: 5318","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.683172475Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"downstairs","":"downstairs","session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3982026-05-20T22:29:45.641Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.683185933Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"":"downstairs","":"downstairs","session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
3992026-05-20T22:29:45.641Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.683195623Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
4002026-05-20T22:29:45.641Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.683204652Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"client":"0","":"downstairs","session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
4012026-05-20T22:29:45.641Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.683214231Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"client":"1","":"downstairs","session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
4022026-05-20T22:29:45.642Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.683228179Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"client":"2","":"downstairs","session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
4032026-05-20T22:29:45.642Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.683238138Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
4042026-05-20T22:29:45.642Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.683251987Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
4052026-05-20T22:29:45.642Z{"msg":"07d4e7f7-7406-48c7-a631-543d3b23baf9 is now active with session: be31a740-d5a1-4f0e-b86a-a8ff0db414d3","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.683262506Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
4062026-05-20T22:29:45.642Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.683275515Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"session_id":"be31a740-d5a1-4f0e-b86a-a8ff0db414d3"}
4072026-05-20T22:29:45.642Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.683285344Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188}
4082026-05-20T22:29:45.642Z{"msg":"Activated sub_volume 07d4e7f7-7406-48c7-a631-543d3b23baf9","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.683297783Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188}
4092026-05-20T22:29:45.642Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.683306172Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"task":"crutest"}
4102026-05-20T22:29:45.642ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
4112026-05-20T22:29:45.642ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4122026-05-20T22:29:45.642Z{"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.68384943Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"task":"crutest"}
4132026-05-20T22:29:45.664ZRead and Verify all blocks (0..100 range:false)
4142026-05-20T22:29:45.711Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.756530311Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"task":"crutest"}
4152026-05-20T22:29:45.711Z{"msg":"CLIENT: Tests done. All submitted work has been ACK'd","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.756553759Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"task----------------------------------------------------------------
4162026-05-20T22:29:45.711Z":"crutest"}
4172026-05-20T22:29:45.711Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4182026-05-20T22:29:45.711ZStates: Active Active Active
4192026-05-20T22:29:45.711ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4202026-05-20T22:29:45.711Z 1002 Acked Read 50 Done Done Done false
4212026-05-20T22:29:45.711Z STATES DS:0 DS:1 DS:2 TOTAL
4222026-05-20T22:29:45.711Z Sent 0 0 0 0
4232026-05-20T22:29:45.711Z Done 1 1 1 3
4242026-05-20T22:29:45.711Z Skipped 0 0 0 0
4252026-05-20T22:29:45.711Z Error 0 0 0 0
4262026-05-20T22:29:45.715ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4272026-05-20T22:29:45.715ZDownstairs last five completed: 1001 1000
4282026-05-20T22:29:45.715ZUpstairs last five completed: 1002 1001 1000
4292026-05-20T22:29:45.715Z----------------------------------------------------------------
4302026-05-20T22:29:45.715Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4312026-05-20T22:29:45.715ZStates: Active Active Active
4322026-05-20T22:29:45.715ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4332026-05-20T22:29:45.715Z 1002 Acked Read 50 Done Done Done false
4342026-05-20T22:29:45.715Z STATES DS:0 DS:1 DS:2 TOTAL
4352026-05-20T22:29:45.715Z Sent 0 0 0 0
4362026-05-20T22:29:45.715Z Done 1 1 1 3
4372026-05-20T22:29:45.715Z Skipped 0 0 0 0
4382026-05-20T22:29:45.715Z Error 0 0 0 0
4392026-05-20T22:29:45.715ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4402026-05-20T22:29:45.715ZDownstairs last five completed: 1001 1000
4412026-05-20T22:29:45.715ZUpstairs last five completed: 1002 1001 1000
4422026-05-20T22:29:45.715Z{"msg":"CLIENT: Up:0 ds:2 act:6","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:44.756651989Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"task":"crutest"}
4432026-05-20T22:29:49.712Z----------------------------------------------------------------
4442026-05-20T22:29:49.713Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4452026-05-20T22:29:49.713ZDownstairs last five completed: 1003 1002 1001 1000
4462026-05-20T22:29:49.713ZUpstairs last five completed: 1003 1002 1001 1000
4472026-05-20T22:29:49.716Z----------------------------------------------------------------
4482026-05-20T22:29:49.716Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4492026-05-20T22:29:49.716ZDownstairs last five completed: 1003 1002 1001 1000
4502026-05-20T22:29:49.716ZUpstairs last five completed: 1003 1002 1001 1000
4512026-05-20T22:29:49.716Z{"msg":"CLIENT: Up:0 ds:0 act:6","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.757972843Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"task":"crutest"}
4522026-05-20T22:29:49.716Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.758038207Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188,"task":"crutest"}
4532026-05-20T22:29:49.716Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-20T22:29:48.758055525Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1188}
4542026-05-20T22:29:49.720ZRepair part 1 passed
4552026-05-20T22:29:49.720Z
4562026-05-20T22:29:49.720ZKill the current downstairs
4572026-05-20T22:29:49.762ZDownstairs 2 stopped
4582026-05-20T22:29:49.762Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
4592026-05-20T22:29:49.765ZNow put back the original so we have a mismatch
4602026-05-20T22:29:49.765Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
4612026-05-20T22:29:49.765ZRestart downstairs with old directory
4622026-05-20T22:29:49.779Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4632026-05-20T22:29:49.790ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4642026-05-20T22:29:49.793Z 0 000-009 32 32 31 4000 4000 3999 F F F <---
4652026-05-20T22:29:49.793Z 1 010-019 32 32 31 4000 4000 3999 F F F <---
4662026-05-20T22:29:49.793Z 2 020-029 32 32 31 4000 4000 3999 F F F <---
4672026-05-20T22:29:49.793Z 3 030-039 32 32 31 4000 4000 3999 F F F <---
4682026-05-20T22:29:49.793Z 4 040-049 32 32 31 4000 4000 3999 F F F <---
4692026-05-20T22:29:49.793ZMax gen: 32, Max flush: 4000
4702026-05-20T22:29:49.793ZError: Difference in extent metadata found!
4712026-05-20T22:29:49.793Zdump test found error as expected
4722026-05-20T22:29:49.793Z
4732026-05-20T22:29:49.793Z
4742026-05-20T22:29:49.793Z/var/tmp/bins/crutest verify --range -g 33 -q --key 99GuRqRlXDbJ63iKGTcYBOwIfPYoIMyQyQAw+iYS0AM= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
4752026-05-20T22:29:49.811Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.855856721Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"task":"crutest"}
4762026-05-20T22:29:49.814Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.856111436Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"task":"crutest"}
4772026-05-20T22:29:49.814Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.856760484Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"task":"crutest"}
4782026-05-20T22:29:49.814Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.857005941Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196}
4792026-05-20T22:29:49.814Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"8a081a6676ca7e9272269457d75f77537f9f2512\",\n git_commit_timestamp: \"2026-05-20T22:09:56.000000000Z\",\n git_branch: \"sunshowers/spr/meta-update-dropshot-api-manager-to-072\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.857052686Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196}
4802026-05-20T22:29:49.814Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.857067175Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196}
4812026-05-20T22:29:49.814Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.857079774Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196}
4822026-05-20T22:29:49.814Z{"msg":"Crucible 05d5802c-ea7d-470b-9c77-b1e1dfd83721 has session id: 5f4a642b-7b38-403c-a024-d563d4fdbcb3","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.857436879Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
4832026-05-20T22:29:49.814Z{"msg":"Upstairs opts: Upstairs UUID: 05d5802c-ea7d-470b-9c77-b1e1dfd83721, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.857459767Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
4842026-05-20T22:29:49.814Z{"msg":"Crucible stats registered with UUID: 05d5802c-ea7d-470b-9c77-b1e1dfd83721","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.857470726Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
4852026-05-20T22:29:49.814Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-20T22:29:48.857480385Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
4862026-05-20T22:29:49.814Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.857634011Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
4872026-05-20T22:29:49.814Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.857658118Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
4882026-05-20T22:29:49.814Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.857668537Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
4892026-05-20T22:29:49.815Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.85848068Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"task":"crutest"}
4902026-05-20T22:29:49.815Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.858544123Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196}
4912026-05-20T22:29:49.815Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"8a081a6676ca7e9272269457d75f77537f9f2512\",\n git_commit_timestamp: \"2026-05-20T22:09:56.000000000Z\",\n git_branch: \"sunshowers/spr/meta-update-dropshot-api-manager-to-072\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.858568291Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196}
4922026-05-20T22:29:49.815Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.85858037Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196}
4932026-05-20T22:29:49.815Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.858592189Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196}
4942026-05-20T22:29:49.815Z{"msg":"Crucible 05d5802c-ea7d-470b-9c77-b1e1dfd83721 has session id: c711c92e-89c4-4400-abc5-bec84ca3f1b8","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.858759643Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
4952026-05-20T22:29:49.815Z{"msg":"Upstairs opts: Upstairs UUID: 05d5802c-ea7d-470b-9c77-b1e1dfd83721, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.858778851Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
4962026-05-20T22:29:49.815Z{"msg":"Crucible stats registered with UUID: 05d5802c-ea7d-470b-9c77-b1e1dfd83721","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.85878911Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
4972026-05-20T22:29:49.815Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-20T22:29:48.858802539Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
4982026-05-20T22:29:49.815Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.858811548Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
4992026-05-20T22:29:49.815Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.858826116Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
5002026-05-20T22:29:49.815Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.858836465Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
5012026-05-20T22:29:49.815Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.858942995Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"task":"crutest"}
5022026-05-20T22:29:49.815Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.858960004Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196}
5032026-05-20T22:29:49.815Z{"msg":"05d5802c-ea7d-470b-9c77-b1e1dfd83721 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.858969813Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5042026-05-20T22:29:49.815Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.858986621Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"client":"0","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5052026-05-20T22:29:49.815Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.85899634Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"client":"1","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5062026-05-20T22:29:49.815Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.859010129Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"client":"2","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5072026-05-20T22:29:49.815Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.859019508Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5082026-05-20T22:29:49.815Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.859033607Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5092026-05-20T22:29:49.815Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.859043346Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5102026-05-20T22:29:49.815Z{"msg":"ds_connection connected from Ok(127.0.0.1:53670)","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.859095491Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5112026-05-20T22:29:49.815Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-05-20T22:29:48.859128907Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5122026-05-20T22:29:49.815Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-05-20T22:29:48.859149925Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5132026-05-20T22:29:49.815Z{"msg":"ds_connection connected from Ok(127.0.0.1:37044)","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.859161264Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5142026-05-20T22:29:49.815Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-05-20T22:29:48.859212779Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5152026-05-20T22:29:49.815Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.859328528Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5162026-05-20T22:29:49.815Z{"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.859367775Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5172026-05-20T22:29:49.819Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.859817462Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"client":"1","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5182026-05-20T22:29:49.819Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.859839239Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"client":"0","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5192026-05-20T22:29:49.819Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.859954998Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5202026-05-20T22:29:49.819Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:48.859969917Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5212026-05-20T22:29:59.816Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.860704767Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5222026-05-20T22:29:59.816Z{"msg":"ds_connection connected from Ok(127.0.0.1:42879)","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.860793998Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5232026-05-20T22:29:59.822Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.86212877Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"client":"2","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5242026-05-20T22:29:59.822Z{"msg":"[0]R flush_numbers: [4000, 4000, 4000, 4000, 4000]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.862461109Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"downstairs","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5252026-05-20T22:29:59.822Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.862494755Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"downstairs","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5262026-05-20T22:29:59.822Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.862520583Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"downstairs","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5272026-05-20T22:29:59.822Z{"msg":"[1]R flush_numbers: [4000, 4000, 4000, 4000, 4000]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.86254935Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"downstairs","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5282026-05-20T22:29:59.822Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.862569038Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"downstairs","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5292026-05-20T22:29:59.822Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.862595906Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"downstairs","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5302026-05-20T22:29:59.822Z{"msg":"[2]R flush_numbers: [3999, 3999, 3999, 3999, 3999]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.862615904Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"downstairs","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5312026-05-20T22:29:59.822Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.862642561Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"downstairs","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5322026-05-20T22:29:59.822Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.862662229Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"downstairs","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5332026-05-20T22:29:59.822Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.862688387Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"downstairs","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5342026-05-20T22:29:59.822Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.862708375Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"downstairs","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5352026-05-20T22:29:59.822Z{"msg":"Next flush: 4001","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.862734282Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"downstairs","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5362026-05-20T22:29:59.822Z{"msg":"extent 0 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3999, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.86276315Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5372026-05-20T22:29:59.822Z{"msg":"extent:0 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3999, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.862800086Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5382026-05-20T22:29:59.823Z{"msg":"found dest for source 0 for extent at index 0 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.862820894Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5392026-05-20T22:29:59.823Z{"msg":"extent 1 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3999, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.862856621Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5402026-05-20T22:29:59.823Z{"msg":"extent:1 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3999, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.862892327Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5412026-05-20T22:29:59.823Z{"msg":"found dest for source 0 for extent at index 1 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.862920485Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5422026-05-20T22:29:59.823Z{"msg":"extent 2 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3999, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.862948962Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5432026-05-20T22:29:59.823Z{"msg":"extent:2 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3999, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.862984628Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5442026-05-20T22:29:59.823Z{"msg":"found dest for source 0 for extent at index 2 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.863014306Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5452026-05-20T22:29:59.823Z{"msg":"extent 3 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3999, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.863045273Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5462026-05-20T22:29:59.823Z{"msg":"extent:3 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3999, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.863079399Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5472026-05-20T22:29:59.823Z{"msg":"found dest for source 0 for extent at index 3 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.863105957Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5482026-05-20T22:29:59.823Z{"msg":"extent 4 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3999, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.863131904Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5492026-05-20T22:29:59.823Z{"msg":"extent:4 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3999, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.863163441Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5502026-05-20T22:29:59.823Z{"msg":"found dest for source 0 for extent at index 4 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.863181989Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5512026-05-20T22:29:59.823Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.863215006Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"client":"0","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5522026-05-20T22:29:59.823Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.863240704Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"client":"1","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5532026-05-20T22:29:59.823Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.863259792Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"client":"2","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5542026-05-20T22:29:59.823Z{"msg":"Full repair list: {ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.863303178Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"downstairs","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5552026-05-20T22:29:59.823Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 4001, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 4001, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 4001, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 4001, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 4001, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.863475661Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"downstairs","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5562026-05-20T22:29:59.823Z{"msg":"starting reconciliation 6a376209-a995-4f31-8c51-16e0737f1a9b: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.863502089Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"downstairs","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5572026-05-20T22:29:59.823Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.863519957Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"client":"2","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5582026-05-20T22:29:59.828Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.872257581Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"client":"2","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5592026-05-20T22:29:59.831Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.874428133Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"client":"2","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5602026-05-20T22:29:59.834Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.876535921Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"client":"2","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5612026-05-20T22:29:59.834Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.878580745Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"client":"2","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5622026-05-20T22:29:59.838Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.88030999Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"downstairs","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5632026-05-20T22:29:59.838Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.880326778Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"client":"0","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5642026-05-20T22:29:59.838Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.880337797Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"client":"1","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5652026-05-20T22:29:59.838Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.880353525Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"client":"2","":"downstairs","session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5662026-05-20T22:29:59.838Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.880364854Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5672026-05-20T22:29:59.838Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.880380423Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5682026-05-20T22:29:59.838Z{"msg":"05d5802c-ea7d-470b-9c77-b1e1dfd83721 is now active with session: 5f4a642b-7b38-403c-a024-d563d4fdbcb3","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.880391862Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"session_id":"5f4a642b-7b38-403c-a024-d563d4fdbcb3"}
5692026-05-20T22:29:59.838Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.88040684Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196}
5702026-05-20T22:29:59.838Z{"msg":"Activated sub_volume 05d5802c-ea7d-470b-9c77-b1e1dfd83721","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.880422069Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196}
5712026-05-20T22:29:59.838Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.880481043Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196}
5722026-05-20T22:29:59.838Z{"msg":"05d5802c-ea7d-470b-9c77-b1e1dfd83721 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.880495352Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
5732026-05-20T22:29:59.838Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.880505511Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"client":"0","":"downstairs","session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
5742026-05-20T22:29:59.838Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.880521079Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"client":"1","":"downstairs","session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
5752026-05-20T22:29:59.838Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.880531828Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"client":"2","":"downstairs","session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
5762026-05-20T22:29:59.839Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.880547347Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
5772026-05-20T22:29:59.839Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.880559216Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
5782026-05-20T22:29:59.839Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.880570255Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
5792026-05-20T22:29:59.839Z{"msg":"ds_connection connected from Ok(127.0.0.1:36057)","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.880600312Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
5802026-05-20T22:29:59.839Z{"msg":"ds_connection connected from Ok(127.0.0.1:44042)","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.88061731Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
5812026-05-20T22:29:59.839Z{"msg":"ds_connection connected from Ok(127.0.0.1:42784)","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.880628949Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
5822026-05-20T22:29:59.839Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.880991764Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"client":"1","":"downstairs","session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
5832026-05-20T22:29:59.839Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.881126261Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"client":"0","":"downstairs","session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
5842026-05-20T22:29:59.839Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.881203694Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
5852026-05-20T22:29:59.839Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.88124576Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"client":"2","":"downstairs","session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
5862026-05-20T22:29:59.839Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.881301385Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
5872026-05-20T22:29:59.839Z{"msg":"[0]R flush_numbers: [5318, 5318, 5318, 5318, 5318]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.881359739Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"downstairs","":"downstairs","session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
5882026-05-20T22:29:59.839Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.881374668Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"downstairs","":"downstairs","session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
5892026-05-20T22:29:59.839Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.881384837Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"downstairs","":"downstairs","session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
5902026-05-20T22:29:59.839Z{"msg":"[1]R flush_numbers: [5318, 5318, 5318, 5318, 5318]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.881399275Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"downstairs","":"downstairs","session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
5912026-05-20T22:29:59.839Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.881409444Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"downstairs","":"downstairs","session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
5922026-05-20T22:29:59.839Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.881420023Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"downstairs","":"downstairs","session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
5932026-05-20T22:29:59.839Z{"msg":"[2]R flush_numbers: [5318, 5318, 5318, 5318, 5318]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.881434282Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"downstairs","":"downstairs","session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
5942026-05-20T22:29:59.839Z{"msg":"[2]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.881444321Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"downstairs","":"downstairs","session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
5952026-05-20T22:29:59.839Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.88145409Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"downstairs","":"downstairs","session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
5962026-05-20T22:29:59.839Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.881467649Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"downstairs","":"downstairs","session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
5972026-05-20T22:29:59.839Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.881477368Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"downstairs","":"downstairs","session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
5982026-05-20T22:29:59.839Z{"msg":"Next flush: 5319","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.881490667Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"downstairs","":"downstairs","session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
5992026-05-20T22:29:59.839Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.881500056Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"":"downstairs","":"downstairs","session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
6002026-05-20T22:29:59.840Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.881513744Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
6012026-05-20T22:29:59.840Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.881522423Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"client":"0","":"downstairs","session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
6022026-05-20T22:29:59.840Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.881536062Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"client":"1","":"downstairs","session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
6032026-05-20T22:29:59.840Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.881545641Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"client":"2","":"downstairs","session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
6042026-05-20T22:29:59.840Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.88155961Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
6052026-05-20T22:29:59.840Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.881568769Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
6062026-05-20T22:29:59.840Z{"msg":"05d5802c-ea7d-470b-9c77-b1e1dfd83721 is now active with session: c711c92e-89c4-4400-abc5-bec84ca3f1b8","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.881582968Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
6072026-05-20T22:29:59.840Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.881591617Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"session_id":"c711c92e-89c4-4400-abc5-bec84ca3f1b8"}
6082026-05-20T22:29:59.840Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.881605136Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196}
6092026-05-20T22:29:59.840Z{"msg":"Activated sub_volume 05d5802c-ea7d-470b-9c77-b1e1dfd83721","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.881613445Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196}
6102026-05-20T22:29:59.840Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.881625664Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"task":"crutest"}
6112026-05-20T22:29:59.840ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
6122026-05-20T22:29:59.840ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
6132026-05-20T22:29:59.840ZRead and Verify all blocks (0..100 range:true)
6142026-05-20T22:29:59.886Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.930287205Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"task":"crutest"}
6152026-05-20T22:29:59.886Z{"msg":"Verify test completed","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.930308592Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"task":"crutest"}
6162026-05-20T22:29:59.890Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.93032956Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"task":"crutest"}
6172026-05-20T22:29:59.890Z{"msg":"CLIENT: Tests done. All submitted work has been ACK'd","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.93033883Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"task":"crutest"}
6182026-05-20T22:29:59.890Z----------------------------------------------------------------
6192026-05-20T22:29:59.890Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6202026-05-20T22:29:59.890ZStates: Active Active Active
6212026-05-20T22:29:59.890ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6222026-05-20T22:29:59.890Z 1000 Acked Read 50 Done Done Done false
6232026-05-20T22:29:59.890Z STATES DS:0 DS:1 DS:2 TOTAL
6242026-05-20T22:29:59.890Z Sent 0 0 0 0
6252026-05-20T22:29:59.890Z Done 1 1 1 3
6262026-05-20T22:29:59.890Z Skipped 0 0 0 0
6272026-05-20T22:29:59.890Z Error 0 0 0 0
6282026-05-20T22:29:59.890ZLast Flush: None None None
6292026-05-20T22:29:59.890ZDownstairs last five completed:
6302026-05-20T22:29:59.890ZUpstairs last five completed: 1000
6312026-05-20T22:29:59.890Z----------------------------------------------------------------
6322026-05-20T22:29:59.890Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6332026-05-20T22:29:59.890ZStates: Active Active Active
6342026-05-20T22:29:59.890ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6352026-05-20T22:29:59.890Z 1000 Acked Read 50 Done Done Done false
6362026-05-20T22:29:59.890Z STATES DS:0 DS:1 DS:2 TOTAL
6372026-05-20T22:29:59.890Z Sent 0 0 0 0
6382026-05-20T22:29:59.890Z Done 1 1 1 3
6392026-05-20T22:29:59.891Z Skipped 0 0 0 0
6402026-05-20T22:29:59.891Z Error 0 0 0 0
6412026-05-20T22:29:59.891ZLast Flush: None None None
6422026-05-20T22:29:59.891ZDownstairs last five completed:
6432026-05-20T22:29:59.891ZUpstairs last five completed: 1000
6442026-05-20T22:29:59.891Z{"msg":"CLIENT: Up:0 ds:2 act:6","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:29:58.930496864Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"task":"crutest"}
6452026-05-20T22:30:03.887Z----------------------------------------------------------------
6462026-05-20T22:30:03.888Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6472026-05-20T22:30:03.888ZDownstairs last five completed: 1001 1000
6482026-05-20T22:30:03.888ZUpstairs last five completed: 1001 1000
6492026-05-20T22:30:03.888Z----------------------------------------------------------------
6502026-05-20T22:30:03.891Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6512026-05-20T22:30:03.891ZDownstairs last five completed: 1001 1000
6522026-05-20T22:30:03.891ZUpstairs last five completed: 1001 1000
6532026-05-20T22:30:03.891Z{"msg":"CLIENT: Up:0 ds:0 act:6","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:30:02.931602738Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"task":"crutest"}
6542026-05-20T22:30:03.891Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-05-20T22:30:02.931668062Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196,"task":"crutest"}
6552026-05-20T22:30:03.891Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-20T22:30:02.931728386Z","hostname":"w-01KS3QV8DJR41ZPDFCSQ6C99N5","pid":1196}
6562026-05-20T22:30:03.894ZRepair part 2 passed
6572026-05-20T22:30:03.894Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
6582026-05-20T22:30:03.907ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6592026-05-20T22:30:03.908Z 0 000-009 32 32 32 4000 4000 4000 F F F
6602026-05-20T22:30:03.908Z 1 010-019 32 32 32 4000 4000 4000 F F F
6612026-05-20T22:30:03.908Z 2 020-029 32 32 32 4000 4000 4000 F F F
6622026-05-20T22:30:03.908Z 3 030-039 32 32 32 4000 4000 4000 F F F
6632026-05-20T22:30:03.908Z 4 040-049 32 32 32 4000 4000 4000 F F F
6642026-05-20T22:30:03.912ZMax gen: 32, Max flush: 4000
6652026-05-20T22:30:03.912Zdump test passed
6662026-05-20T22:30:03.912Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
6672026-05-20T22:30:03.922Z Extent 1
6682026-05-20T22:30:03.925ZGEN 32 32 32
6692026-05-20T22:30:03.925ZFLUSH_ID 4000 4000 4000
6702026-05-20T22:30:03.925ZDIRTY
6712026-05-20T22:30:03.925Z
6722026-05-20T22:30:03.926ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
6732026-05-20T22:30:03.926Z010 A A A A A A
6742026-05-20T22:30:03.926Z011 A A A A A A
6752026-05-20T22:30:03.926Z012 A A A A A A
6762026-05-20T22:30:03.926Z013 A A A A A A
6772026-05-20T22:30:03.926Z014 A A A A A A
6782026-05-20T22:30:03.926Z015 A A A A A A
6792026-05-20T22:30:03.929Z016 A A A A A A
6802026-05-20T22:30:03.929Z017 A A A A A A
6812026-05-20T22:30:03.929Z018 A A A A A A
6822026-05-20T22:30:03.929Z019 A A A A A A
6832026-05-20T22:30:03.932Zdump extent test passed
6842026-05-20T22:30:03.932Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
6852026-05-20T22:30:03.940ZExtent 2 Block in extent 0 Actual block 20
6862026-05-20T22:30:03.940Z
6872026-05-20T22:30:03.943Z DATA SHA256 VER
6882026-05-20T22:30:03.943Z------ ---------------------------------------------------------------- ---
6892026-05-20T22:30:03.943Z 0 f47237b2dd182e60c94be921f9757a7d190c69e17c6200ca804f979727ad9ca3 A
6902026-05-20T22:30:03.943Z 1 f47237b2dd182e60c94be921f9757a7d190c69e17c6200ca804f979727ad9ca3 A
6912026-05-20T22:30:03.943Z 2 f47237b2dd182e60c94be921f9757a7d190c69e17c6200ca804f979727ad9ca3 A
6922026-05-20T22:30:03.943Z
6932026-05-20T22:30:03.943ZNONCES 0 1 2 DIFF
6942026-05-20T22:30:03.943Z------ ------------------------ ------------------------ ------------------------ -----
6952026-05-20T22:30:03.943Z 0 acf904565ecdc78575e3e7b7 acf904565ecdc78575e3e7b7 acf904565ecdc78575e3e7b7
6962026-05-20T22:30:03.943Z
6972026-05-20T22:30:03.943Z TAGS 0 1 2 DIFF
6982026-05-20T22:30:03.943Z------ -------------------------------- -------------------------------- -------------------------------- -----
6992026-05-20T22:30:03.943Z 0 10ab90c3fe43747e1e1c886b2f43b32b 10ab90c3fe43747e1e1c886b2f43b32b 10ab90c3fe43747e1e1c886b2f43b32b
7002026-05-20T22:30:03.944Z
7012026-05-20T22:30:03.944ZHASHES 0 1 2 DIFF
7022026-05-20T22:30:03.944Z------ ---------------- ---------------- ---------------- -----
7032026-05-20T22:30:03.944Z
7042026-05-20T22:30:03.947Zdump block test passed
7052026-05-20T22:30:03.947ZInitial upstairs tests have completed, stopping all downstairs
7062026-05-20T22:30:08.967ZCreating 7 larger downstairs regions
7072026-05-20T22:30:09.176ZStarting 7 downstairs
7082026-05-20T22:30:14.182Zdsc restarted at PID: 1212
7092026-05-20T22:30:14.280ZNow do the replace-reconcile test
7102026-05-20T22:30:14.294ZUsing 8870 for the replacement port
7112026-05-20T22:33:47.081ZCompleted test: replace-reconcile
7122026-05-20T22:33:47.084ZNow do the replace-before-active test
7132026-05-20T22:37:42.421ZCompleted test: replace-before-active
7142026-05-20T22:37:42.421ZAll tests have completed, stopping all downstairs
7152026-05-20T22:37:42.439Z
7162026-05-20T22:37:42.439ZAll Tests have passed
7172026-05-20T22:37:42.439Z11:39 Test duration
7182026-05-20T22:37:42.439Z
7192026-05-20T22:37:42.439Zreal 11:38.607881358
7202026-05-20T22:37:42.439Zuser 17:06.003142764
7212026-05-20T22:37:42.439Zsys 11.884273700
7222026-05-20T22:37:42.439Ztrap 0.432269742
7232026-05-20T22:37:42.439Ztflt 0.049114727
7242026-05-20T22:37:42.439Zdflt 0.029421220
7252026-05-20T22:37:42.439Zkflt 0.008484665
7262026-05-20T22:37:42.439Zlock 16:02:10.636234003
7272026-05-20T22:37:42.440Zslp 50:31.591283659
7282026-05-20T22:37:42.440Zlat 1:53.601932263
7292026-05-20T22:37:42.440Zstop 0.052719212
7302026-05-20T22:37:42.443Z+ echo 'test-up-2region-encrypted ends'
7312026-05-20T22:37:42.443Ztest-up-2region-encrypted ends
7322026-05-20T22:37:47.443Zprocess exited: duration 707749 ms, exit code 0
7332026-05-20T22:37:47.443Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
7342026-05-20T22:38:47.483Zexec warning: : stdout descriptor may be held open by a background process; giving up!
7352026-05-20T22:38:47.483Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
7362026-05-20T22:38:47.495Zfound 17 output files
7372026-05-20T22:38:47.495Zuploading: /tmp/test_up-build/dsc-out.txt (58591 bytes)
7382026-05-20T22:38:48.514Zuploaded: /tmp/test_up-build/dsc-out.txt
7392026-05-20T22:38:48.514Zuploading: /tmp/test_up-build/test_up_out.txt (23127926 bytes)
7402026-05-20T22:38:50.121Zuploaded: /tmp/test_up-build/test_up_out.txt
7412026-05-20T22:38:50.121Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (551027 bytes)
7422026-05-20T22:38:51.168Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
7432026-05-20T22:38:51.168Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (548341 bytes)
7442026-05-20T22:38:52.194Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
7452026-05-20T22:38:52.194Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (978696 bytes)
7462026-05-20T22:38:53.215Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
7472026-05-20T22:38:53.215Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (543090 bytes)
7482026-05-20T22:38:54.250Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
7492026-05-20T22:38:54.250Zuploading: /tmp/test_up-build/dsc/downstairs-8850.txt (2529 bytes)
7502026-05-20T22:38:55.263Zuploaded: /tmp/test_up-build/dsc/downstairs-8850.txt
7512026-05-20T22:38:55.267Zuploading: /tmp/test_up-build/dsc/downstairs-8860.txt (47366 bytes)
7522026-05-20T22:38:56.277Zuploaded: /tmp/test_up-build/dsc/downstairs-8860.txt
7532026-05-20T22:38:56.278Zuploading: /tmp/test_up-build/dsc/downstairs-8870.txt (3577694 bytes)
7542026-05-20T22:38:57.315Zuploaded: /tmp/test_up-build/dsc/downstairs-8870.txt
7552026-05-20T22:38:57.315Zuploading: /tmp/debug/df.txt (1270 bytes)
7562026-05-20T22:38:58.325Zuploaded: /tmp/debug/df.txt
7572026-05-20T22:38:58.326Zuploading: /tmp/debug/dtrace.txt (1041650 bytes)
7582026-05-20T22:38:59.351Zuploaded: /tmp/debug/dtrace.txt
7592026-05-20T22:38:59.351Zuploading: /tmp/debug/iostat.txt (226413 bytes)
7602026-05-20T22:38:59.362Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 226413 -> 229965
7612026-05-20T22:39:00.400Zuploaded: /tmp/debug/iostat.txt
7622026-05-20T22:39:00.401Zuploading: /tmp/debug/mpstat.txt (554078 bytes)
7632026-05-20T22:39:00.430Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 554078 -> 563503
7642026-05-20T22:39:01.475Zuploaded: /tmp/debug/mpstat.txt
7652026-05-20T22:39:01.475Zuploading: /tmp/debug/paging.txt (96970 bytes)
7662026-05-20T22:39:01.501Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 96970 -> 98682
7672026-05-20T22:39:02.606Zuploaded: /tmp/debug/paging.txt
7682026-05-20T22:39:02.607Zuploading: /tmp/debug/prstat.txt (1064207 bytes)
7692026-05-20T22:39:02.641Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 1064207 -> 1083550
7702026-05-20T22:39:03.667Zuploaded: /tmp/debug/prstat.txt
7712026-05-20T22:39:03.667Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
7722026-05-20T22:39:04.715Zuploaded: /tmp/debug/psrinfo.txt
7732026-05-20T22:39:04.715Zuploading: /tmp/debug/upstairs-info.txt (164339 bytes)
7742026-05-20T22:39:05.766Zuploaded: /tmp/debug/upstairs-info.txt