01KY0V34S8DCMZD212E0Z1D4ZV: test-up-2region-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01KY0V3QVNBRZDW3P04V86QTMB

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12026-07-20T22:54:41.900Zjob dependencies complete; ready to run (waiting for 12 m 16 s)
22026-07-20T22:55:42.203Zjob assigned to worker 01KY0VTGNPBVHPAV3TNY4P3W1H [factory aws, i-00909958531fa17a8] (queued for 1 m 0 s)
32026-07-20T22:55:49.506Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42026-07-20T22:55:51.402Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52026-07-20T22:55:51.406Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62026-07-20T22:55:52.997Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72026-07-20T22:55:53.002Zdownloading input: /input/build/work/bins/crutest.gz
82026-07-20T22:55:54.739Zdownloaded input: /input/build/work/bins/crutest.gz
92026-07-20T22:55:54.759Zdownloading input: /input/build/work/bins/dsc.gz
102026-07-20T22:55:55.556Zdownloaded input: /input/build/work/bins/dsc.gz
112026-07-20T22:55:55.563Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122026-07-20T22:55:55.582Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132026-07-20T22:55:55.582Zdownloading input: /input/build/work/scripts/test_ds.sh
142026-07-20T22:55:55.617Zdownloaded input: /input/build/work/scripts/test_ds.sh
152026-07-20T22:55:55.617Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162026-07-20T22:55:55.648Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172026-07-20T22:55:55.648Zdownloading input: /input/build/work/scripts/test_repair.sh
182026-07-20T22:55:55.676Zdownloaded input: /input/build/work/scripts/test_repair.sh
192026-07-20T22:55:55.695Zdownloading input: /input/build/work/scripts/test_replay.sh
202026-07-20T22:55:55.705Zdownloaded input: /input/build/work/scripts/test_replay.sh
212026-07-20T22:55:55.705Zdownloading input: /input/build/work/scripts/test_up.sh
222026-07-20T22:55:55.734Zdownloaded input: /input/build/work/scripts/test_up.sh
232026-07-20T22:55:55.740Zdownloading input: /input/build/work/scripts/upstairs_info.d
242026-07-20T22:55:55.762Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252026-07-20T22:55:55.762Zdownloading input: /input/build/tmp/cargo-test-out.log
262026-07-20T22:55:56.302Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272026-07-20T22:55:56.315Zstarting task 0: "setup"
282026-07-20T22:55:56.320Z++ uname -s
292026-07-20T22:55:56.320Z+ kern=SunOS
302026-07-20T22:55:56.320Z+ build_user=build
312026-07-20T22:55:56.320Z+ build_uid=12345
322026-07-20T22:55:56.320Z+ work_dir=/work
332026-07-20T22:55:56.320Z+ input_dir=/input
342026-07-20T22:55:56.321Z+ [[ 0 == 12345 ]]
352026-07-20T22:55:56.321Z+ case "$kern" in
362026-07-20T22:55:56.321Z+ groupadd -g 12345 build
372026-07-20T22:55:56.321Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382026-07-20T22:55:58.318Z+ zfs create -o mountpoint=/work rpool/work
392026-07-20T22:55:58.710Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402026-07-20T22:55:58.713Z+ home_fs=zfs
412026-07-20T22:55:58.713Z+ [[ zfs == autofs ]]
422026-07-20T22:55:58.713Z+ mkdir -p /home/build
432026-07-20T22:55:58.713Z+ chown build:build /home/build /work
442026-07-20T22:56:00.715Z+ chmod 0700 /home/build /work
452026-07-20T22:56:00.718Zprocess exited: duration 4414 ms, exit code 0
 
462026-07-20T22:56:00.724Zstarting task 1: "authentication"
472026-07-20T22:56:00.742Zprocess exited: duration 17 ms, exit code 0
 
482026-07-20T22:56:00.748Zstarting task 2: "build"
492026-07-20T22:56:00.751Z+ banner cores
502026-07-20T22:56:00.754Z
512026-07-20T22:56:00.754Z #### #### ##### ###### ####
522026-07-20T22:56:00.754Z # # # # # # # #
532026-07-20T22:56:00.754Z # # # # # ##### ####
542026-07-20T22:56:00.754Z # # # ##### # #
552026-07-20T22:56:00.754Z # # # # # # # # #
562026-07-20T22:56:00.754Z #### #### # # ###### ####
572026-07-20T22:56:00.754Z
582026-07-20T22:56:00.754Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
592026-07-20T22:56:00.759Z+ banner unpack
602026-07-20T22:56:00.762Z
612026-07-20T22:56:00.762Z # # # # ##### ## #### # #
622026-07-20T22:56:00.762Z # # ## # # # # # # # # #
632026-07-20T22:56:00.762Z # # # # # # # # # # ####
642026-07-20T22:56:00.762Z # # # # # ##### ###### # # #
652026-07-20T22:56:00.762Z # # # ## # # # # # # #
662026-07-20T22:56:00.762Z #### # # # # # #### # #
672026-07-20T22:56:00.762Z
682026-07-20T22:56:00.762Z+ mkdir -p /var/tmp/bins
692026-07-20T22:56:00.762Z+ for t in "$input/bins/"*.gz
702026-07-20T22:56:00.765Z++ basename /input/build/work/bins/crucible-downstairs.gz
712026-07-20T22:56:00.765Z+ b=crucible-downstairs.gz
722026-07-20T22:56:00.765Z+ b=crucible-downstairs
732026-07-20T22:56:00.765Z+ gunzip
742026-07-20T22:56:03.335Z+ chmod +x /var/tmp/bins/crucible-downstairs
752026-07-20T22:56:03.339Z+ for t in "$input/bins/"*.gz
762026-07-20T22:56:03.339Z++ basename /input/build/work/bins/crucible-hammer.gz
772026-07-20T22:56:03.339Z+ b=crucible-hammer.gz
782026-07-20T22:56:03.339Z+ b=crucible-hammer
792026-07-20T22:56:03.339Z+ gunzip
802026-07-20T22:56:05.748Z+ chmod +x /var/tmp/bins/crucible-hammer
812026-07-20T22:56:05.751Z+ for t in "$input/bins/"*.gz
822026-07-20T22:56:05.751Z++ basename /input/build/work/bins/crutest.gz
832026-07-20T22:56:05.754Z+ b=crutest.gz
842026-07-20T22:56:05.754Z+ b=crutest
852026-07-20T22:56:05.754Z+ gunzip
862026-07-20T22:56:08.289Z+ chmod +x /var/tmp/bins/crutest
872026-07-20T22:56:08.293Z+ for t in "$input/bins/"*.gz
882026-07-20T22:56:08.293Z++ basename /input/build/work/bins/dsc.gz
892026-07-20T22:56:08.293Z+ b=dsc.gz
902026-07-20T22:56:08.293Z+ b=dsc
912026-07-20T22:56:08.293Z+ gunzip
922026-07-20T22:56:09.427Z+ chmod +x /var/tmp/bins/dsc
932026-07-20T22:56:09.431Z+ export BINDIR=/var/tmp/bins
942026-07-20T22:56:09.431Z+ BINDIR=/var/tmp/bins
952026-07-20T22:56:09.431Z+ jobpid=1091
962026-07-20T22:56:09.431Z+ echo 'Setup debug logging'
972026-07-20T22:56:09.431ZSetup debug logging
982026-07-20T22:56:09.431Z+ mkdir /tmp/debug
992026-07-20T22:56:09.431Z+ sleep 3600
1002026-07-20T22:56:09.432Z+ psrinfo -v
1012026-07-20T22:56:09.435Z+ df -h
1022026-07-20T22:56:09.439Z+ prstat -d d -mLc 1
1032026-07-20T22:56:09.442Z+ iostat -T d -xn 1
1042026-07-20T22:56:09.442Z+ mpstat -T d 1
1052026-07-20T22:56:09.442Z+ vmstat -T d -p 1
1062026-07-20T22:56:09.442Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1072026-07-20T22:56:09.442Z+ banner test_up_2r_encrypted
1082026-07-20T22:56:09.442Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1092026-07-20T22:56:09.445Z #####
1102026-07-20T22:56:09.445Z ##### ###### #### ##### # # ##### # # #####
1112026-07-20T22:56:09.445Z # # # # # # # # # # #
1122026-07-20T22:56:09.445Z # ##### #### # # # # # ##### # #
1132026-07-20T22:56:09.445Z # # # # # # ##### # #####
1142026-07-20T22:56:09.446Z # # # # # # # # # # #
1152026-07-20T22:56:09.446Z # ###### #### # ####### #### # ####### ####### # #
1162026-07-20T22:56:09.446Z
1172026-07-20T22:56:09.446Z+ ptime -m bash /input/build/work/scripts/test_up.sh -r 2 -N encrypted
1182026-07-20T22:56:09.451Z/input/build/work
1192026-07-20T22:56:09.459ZUsing 2 region sets
1202026-07-20T22:56:09.462ZTurn off color for downstairs dump
1212026-07-20T22:56:09.471ZUpstairs using key: 78/DNdfG3j2qF4kZzjqDtSDoFKLxr3veIRmQ+njh7UY=
1222026-07-20T22:56:09.475Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1232026-07-20T22:56:09.475ZCreating 6 downstairs regions
1242026-07-20T22:56:09.723ZStarting 6 downstairs
1252026-07-20T22:56:09.726Zdsc started at PID: 1139
1262026-07-20T22:56:14.813ZDisable automatic restart on all downstairs
1272026-07-20T22:56:14.827Z
1282026-07-20T22:56:14.827ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1292026-07-20T22:56:14.831ZRunning test: span
1302026-07-20T22:56:18.907ZCompleted test: span
1312026-07-20T22:56:23.913ZRunning test: big
1322026-07-20T22:56:28.224ZCompleted test: big
1332026-07-20T22:56:33.232ZRunning test: dep
1342026-07-20T22:56:52.921ZCompleted test: dep
1352026-07-20T22:56:57.928ZRunning test: balloon
1362026-07-20T22:59:26.163ZCompleted test: balloon
1372026-07-20T22:59:31.170ZRunning test: deactivate
1382026-07-20T22:59:35.522ZCompleted test: deactivate
1392026-07-20T22:59:40.529ZRunning hammer
1402026-07-20T22:59:52.746ZRun repair tests
1412026-07-20T22:59:52.750Z/var/tmp/bins/crutest fill -g 31 -q --key 78/DNdfG3j2qF4kZzjqDtSDoFKLxr3veIRmQ+njh7UY= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1422026-07-20T22:59:52.768Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.775944577Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"task":"crutest"}
1432026-07-20T22:59:52.774Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.776205896Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"task":"crutest"}
1442026-07-20T22:59:52.774Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.77693808Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"task":"crutest"}
1452026-07-20T22:59:52.775Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.777201209Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182}
1462026-07-20T22:59:52.775Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a81b42ee266cbdd6371f13d7b3f8dd00b79d3cab\",\n git_commit_timestamp: \"2026-07-20T22:41:44.000000000Z\",\n git_branch: \"renovate/crate-rand-vulnerability\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.777230906Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182}
1472026-07-20T22:59:52.775Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.777246004Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182}
1482026-07-20T22:59:52.775Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.777259612Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182}
1492026-07-20T22:59:52.775Z{"msg":"Crucible 645da182-f504-41e1-8d64-af46c4650d6e has session id: c2878676-c36d-487f-968e-868ac7c6ba09","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.777671984Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1502026-07-20T22:59:52.775Z{"msg":"Upstairs opts: Upstairs UUID: 645da182-f504-41e1-8d64-af46c4650d6e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.777733027Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1512026-07-20T22:59:52.775Z{"msg":"Crucible stats registered with UUID: 645da182-f504-41e1-8d64-af46c4650d6e","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.777782281Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1522026-07-20T22:59:52.775Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-20T22:59:51.777808458Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1532026-07-20T22:59:52.775Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.777895388Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1542026-07-20T22:59:52.776Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.777925334Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1552026-07-20T22:59:52.776Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.777948052Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1562026-07-20T22:59:52.776Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.778917888Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"task":"crutest"}
1572026-07-20T22:59:52.776Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.778974241Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182}
1582026-07-20T22:59:52.776Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a81b42ee266cbdd6371f13d7b3f8dd00b79d3cab\",\n git_commit_timestamp: \"2026-07-20T22:41:44.000000000Z\",\n git_branch: \"renovate/crate-rand-vulnerability\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.778998628Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182}
1592026-07-20T22:59:52.776Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.779011437Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182}
1602026-07-20T22:59:52.776Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.779022985Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182}
1612026-07-20T22:59:52.776Z{"msg":"Crucible 645da182-f504-41e1-8d64-af46c4650d6e has session id: dfba38b6-0aa0-4162-8e44-9ea14858bd42","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.7792376Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
1622026-07-20T22:59:52.776Z{"msg":"Upstairs opts: Upstairs UUID: 645da182-f504-41e1-8d64-af46c4650d6e, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.779267926Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
1632026-07-20T22:59:52.776Z{"msg":"Crucible stats registered with UUID: 645da182-f504-41e1-8d64-af46c4650d6e","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.779278715Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
1642026-07-20T22:59:52.776Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-20T22:59:51.779288534Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
1652026-07-20T22:59:52.776Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.779301573Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
1662026-07-20T22:59:52.776Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.779312001Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
1672026-07-20T22:59:52.776Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.77932658Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
1682026-07-20T22:59:52.776Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.779446406Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"task":"crutest"}
1692026-07-20T22:59:52.776Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.779460314Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182}
1702026-07-20T22:59:52.776Z{"msg":"645da182-f504-41e1-8d64-af46c4650d6e active request set","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.779539055Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1712026-07-20T22:59:52.776Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.779553023Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"client":"0","":"downstairs","session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1722026-07-20T22:59:52.776Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.779562772Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"client":"1","":"downstairs","session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1732026-07-20T22:59:52.776Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.77958248Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"client":"2","":"downstairs","session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1742026-07-20T22:59:52.776Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.779592888Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1752026-07-20T22:59:52.776Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.779607707Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1762026-07-20T22:59:52.777Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.779617795Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1772026-07-20T22:59:52.782Z{"msg":"ds_connection connected from Ok(127.0.0.1:61410)","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.779683838Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1782026-07-20T22:59:52.782Z{"msg":"ds_connection connected from Ok(127.0.0.1:61262)","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.779728552Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1792026-07-20T22:59:52.782Z{"msg":"ds_connection connected from Ok(127.0.0.1:51172)","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.779754649Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1802026-07-20T22:59:52.782Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.780534808Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"client":"0","":"downstairs","session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1812026-07-20T22:59:52.782Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.780561814Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"client":"2","":"downstairs","session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1822026-07-20T22:59:52.783Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.780574953Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"client":"1","":"downstairs","session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1832026-07-20T22:59:52.783Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.780778569Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1842026-07-20T22:59:52.783Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.780825414Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1852026-07-20T22:59:52.783Z{"msg":"[0]R flush_numbers: [3994, 3994, 3994, 3994, 3994]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.78085506Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"downstairs","":"downstairs","session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1862026-07-20T22:59:52.783Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.780870048Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"downstairs","":"downstairs","session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1872026-07-20T22:59:52.783Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.780880217Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"downstairs","":"downstairs","session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1882026-07-20T22:59:52.783Z{"msg":"[1]R flush_numbers: [3994, 3994, 3994, 3994, 3994]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.780894355Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"downstairs","":"downstairs","session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1892026-07-20T22:59:52.783Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.780908704Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"downstairs","":"downstairs","session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1902026-07-20T22:59:52.783Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.780919232Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"downstairs","":"downstairs","session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1912026-07-20T22:59:52.783Z{"msg":"[2]R flush_numbers: [3994, 3994, 3994, 3994, 3994]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.780933581Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"downstairs","":"downstairs","session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1922026-07-20T22:59:52.783Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.78094401Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"downstairs","":"downstairs","session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1932026-07-20T22:59:52.783Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.780958428Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"downstairs","":"downstairs","session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1942026-07-20T22:59:52.783Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.780968277Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"downstairs","":"downstairs","session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1952026-07-20T22:59:52.783Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.780982485Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"downstairs","":"downstairs","session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1962026-07-20T22:59:52.783Z{"msg":"Next flush: 3995","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.780992054Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"downstairs","":"downstairs","session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1972026-07-20T22:59:52.783Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781006012Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"downstairs","":"downstairs","session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1982026-07-20T22:59:52.783Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781016201Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
1992026-07-20T22:59:52.783Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.78102946Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"client":"0","":"downstairs","session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
2002026-07-20T22:59:52.783Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781039348Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"client":"1","":"downstairs","session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
2012026-07-20T22:59:52.783Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781049227Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"client":"2","":"downstairs","session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
2022026-07-20T22:59:52.783Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781059096Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
2032026-07-20T22:59:52.783Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781068895Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
2042026-07-20T22:59:52.784Z{"msg":"645da182-f504-41e1-8d64-af46c4650d6e is now active with session: c2878676-c36d-487f-968e-868ac7c6ba09","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781083333Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
2052026-07-20T22:59:52.784Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781092102Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"session_id":"c2878676-c36d-487f-968e-868ac7c6ba09"}
2062026-07-20T22:59:52.784Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781101981Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182}
2072026-07-20T22:59:52.784Z{"msg":"Activated sub_volume 645da182-f504-41e1-8d64-af46c4650d6e","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.78111055Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182}
2082026-07-20T22:59:52.784Z{"msg":"645da182-f504-41e1-8d64-af46c4650d6e active request set","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781118759Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2092026-07-20T22:59:52.784Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781127978Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182}
2102026-07-20T22:59:52.784Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781135797Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"client":"0","":"downstairs","session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2112026-07-20T22:59:52.784Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781145626Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"client":"1","":"downstairs","session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2122026-07-20T22:59:52.784Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781188381Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"client":"2","":"downstairs","session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2132026-07-20T22:59:52.784Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781233466Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2142026-07-20T22:59:52.784Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781268391Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2152026-07-20T22:59:52.784Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.78127895Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2162026-07-20T22:59:52.784Z{"msg":"ds_connection connected from Ok(127.0.0.1:56761)","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781295058Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2172026-07-20T22:59:52.784Z{"msg":"ds_connection connected from Ok(127.0.0.1:39344)","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781309917Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2182026-07-20T22:59:52.784Z{"msg":"ds_connection connected from Ok(127.0.0.1:62434)","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781323675Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2192026-07-20T22:59:52.784Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.78153564Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"client":"1","":"downstairs","session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2202026-07-20T22:59:52.784Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781580945Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"client":"0","":"downstairs","session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2212026-07-20T22:59:52.784Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781606832Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"client":"2","":"downstairs","session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2222026-07-20T22:59:52.784Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781698241Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2232026-07-20T22:59:52.784Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781718949Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2242026-07-20T22:59:52.784Z{"msg":"[0]R flush_numbers: [5313, 5311, 5315, 5315, 5315]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781767353Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"downstairs","":"downstairs","session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2252026-07-20T22:59:52.784Z{"msg":"[0]R generation: [13, 12, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781782111Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"downstairs","":"downstairs","session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2262026-07-20T22:59:52.784Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.78179252Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"downstairs","":"downstairs","session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2272026-07-20T22:59:52.784Z{"msg":"[1]R flush_numbers: [5313, 5311, 5315, 5315, 5315]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781807158Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"downstairs","":"downstairs","session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2282026-07-20T22:59:52.784Z{"msg":"[1]R generation: [13, 12, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781817217Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"downstairs","":"downstairs","session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2292026-07-20T22:59:52.785Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781827796Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"downstairs","":"downstairs","session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2302026-07-20T22:59:52.785Z{"msg":"[2]R flush_numbers: [5313, 5311, 5315, 5315, 5315]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781842134Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"downstairs","":"downstairs","session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2312026-07-20T22:59:52.785Z{"msg":"[2]R generation: [13, 12, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781852443Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"downstairs","":"downstairs","session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2322026-07-20T22:59:52.785Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781866341Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"downstairs","":"downstairs","session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2332026-07-20T22:59:52.785Z{"msg":"Max found gen is 15","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.78187629Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"downstairs","":"downstairs","session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2342026-07-20T22:59:52.785Z{"msg":"Generation requested: 31 >= found:15","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781890538Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"downstairs","":"downstairs","session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2352026-07-20T22:59:52.785Z{"msg":"Next flush: 5316","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781900337Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"downstairs","":"downstairs","session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2362026-07-20T22:59:52.785Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781914246Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"":"downstairs","":"downstairs","session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2372026-07-20T22:59:52.785Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781924454Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2382026-07-20T22:59:52.785Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781937653Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"client":"0","":"downstairs","session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2392026-07-20T22:59:52.785Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781947452Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"client":"1","":"downstairs","session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2402026-07-20T22:59:52.785Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.78196143Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"client":"2","":"downstairs","session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2412026-07-20T22:59:52.785Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781971589Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2422026-07-20T22:59:52.785Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781985237Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2432026-07-20T22:59:52.785Z{"msg":"645da182-f504-41e1-8d64-af46c4650d6e is now active with session: dfba38b6-0aa0-4162-8e44-9ea14858bd42","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.781995736Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2442026-07-20T22:59:52.785Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.782008724Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"session_id":"dfba38b6-0aa0-4162-8e44-9ea14858bd42"}
2452026-07-20T22:59:52.785Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.782018233Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182}
2462026-07-20T22:59:52.785Z{"msg":"Activated sub_volume 645da182-f504-41e1-8d64-af46c4650d6e","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.782030712Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182}
2472026-07-20T22:59:52.785Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.782039011Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"task":"crutest"}
2482026-07-20T22:59:52.785ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
2492026-07-20T22:59:52.785Z{"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.782193363Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"task":"crutest"}
2502026-07-20T22:59:52.799ZRead and Verify all blocks (0..100 range:false)
2512026-07-20T22:59:52.848Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30----------------------------------------------------------------
2522026-07-20T22:59:52.848Z,"time":"2026-07-20T22:59:51.855241311Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H", Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2532026-07-20T22:59:52.848Z"States:pid": Active1182 Active Active
2542026-07-20T22:59:52.848Z,"JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2552026-07-20T22:59:52.848Ztask":"crutest"}
2562026-07-20T22:59:52.848Z 1002 Acked Read 50{ Done" Donemsg Done": false"
2572026-07-20T22:59:52.848ZCLIENT: Tests done. All submitted work has been ACK'd","v":0,"name":" STATES DS:0 DS:1 DS:2 TOTAL
2582026-07-20T22:59:52.848Zcrucible" Sent ," 0 level": 0 30 0 0
2592026-07-20T22:59:52.848Z Done 1 1 1 3
2602026-07-20T22:59:52.848Z, Skipped " 0 time 0 " 0 : 0
2612026-07-20T22:59:52.848Z" Error 0 0 0 0
2622026-07-20T22:59:52.852Z2026-07-20T22:59:51.855277816Z"Last Flush: ,"hostnameSome(JobId(1001)) ":Some(JobId(1001)) "Some(JobId(1001))
2632026-07-20T22:59:52.852Zw-01KY0VTGNPBVHPAV3TNY4P3W1HDownstairs last five completed:","pid":1182,"task": 1001" 1000
2642026-07-20T22:59:52.852Zcrutest"}
2652026-07-20T22:59:52.852ZUpstairs last five completed: 1002 1001 1000
2662026-07-20T22:59:52.852Z----------------------------------------------------------------
2672026-07-20T22:59:52.852Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2682026-07-20T22:59:52.852ZStates: Active Active Active
2692026-07-20T22:59:52.852ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2702026-07-20T22:59:52.853Z 1002 Acked Read 50 Done Done Done false
2712026-07-20T22:59:52.853Z STATES DS:0 DS:1 DS:2 TOTAL
2722026-07-20T22:59:52.853Z Sent 0 0 0 0
2732026-07-20T22:59:52.853Z Done 1 1 1 3
2742026-07-20T22:59:52.853Z Skipped 0 0 0 0
2752026-07-20T22:59:52.853Z Error 0 0 0 0
2762026-07-20T22:59:52.853ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2772026-07-20T22:59:52.853ZDownstairs last five completed: 1001 1000
2782026-07-20T22:59:52.853ZUpstairs last five completed: 1002 1001 1000
2792026-07-20T22:59:52.853Z{"msg":"CLIENT: Up:0 ds:2 act:6","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:51.855377605Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"task":"crutest"}
2802026-07-20T22:59:56.848Z----------------------------------------------------------------
2812026-07-20T22:59:56.848Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2822026-07-20T22:59:56.848ZDownstairs last five completed: 1003 1002 1001 1000
2832026-07-20T22:59:56.848ZUpstairs last five completed: 1003 1002 1001 1000
2842026-07-20T22:59:56.848Z----------------------------------------------------------------
2852026-07-20T22:59:56.848Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2862026-07-20T22:59:56.848ZDownstairs last five completed: 1003 1002 1001 1000
2872026-07-20T22:59:56.848ZUpstairs last five completed: 1003 1002 1001 1000
2882026-07-20T22:59:56.851Z{"msg":"CLIENT: Up:0 ds:0 act:6","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.855993173Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"task":"crutest"}
2892026-07-20T22:59:56.851Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.856055045Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182,"task":"crutest"}
2902026-07-20T22:59:56.851Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-20T22:59:55.856071583Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1182}
2912026-07-20T22:59:56.856ZRepair setup passed
2922026-07-20T22:59:56.859ZCopy the region for /var/tmp/test_up-build/8830
2932026-07-20T22:59:56.859Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2942026-07-20T22:59:56.862Z/var/tmp/bins/crutest fill -g 32 -q --key 78/DNdfG3j2qF4kZzjqDtSDoFKLxr3veIRmQ+njh7UY= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2952026-07-20T22:59:56.880Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.888673123Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"task":"crutest"}
2962026-07-20T22:59:56.885Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.888965969Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"task":"crutest"}
2972026-07-20T22:59:56.885Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.889694593Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"task":"crutest"}
2982026-07-20T22:59:56.885Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.889960882Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188}
2992026-07-20T22:59:56.885Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a81b42ee266cbdd6371f13d7b3f8dd00b79d3cab\",\n git_commit_timestamp: \"2026-07-20T22:41:44.000000000Z\",\n git_branch: \"renovate/crate-rand-vulnerability\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.889990598Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188}
3002026-07-20T22:59:56.885Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.890003467Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188}
3012026-07-20T22:59:56.885Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.890015525Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188}
3022026-07-20T22:59:56.885Z{"msg":"Crucible 03ed9c33-2fa4-4d5d-b97e-1062f2447f49 has session id: f75f48f2-16f1-421c-94c5-bd245e244622","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.890463573Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3032026-07-20T22:59:56.885Z{"msg":"Upstairs opts: Upstairs UUID: 03ed9c33-2fa4-4d5d-b97e-1062f2447f49, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.890507928Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3042026-07-20T22:59:56.885Z{"msg":"Crucible stats registered with UUID: 03ed9c33-2fa4-4d5d-b97e-1062f2447f49","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.890536674Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3052026-07-20T22:59:56.885Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-20T22:59:55.890552892Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3062026-07-20T22:59:56.885Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.890692006Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3072026-07-20T22:59:56.885Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.890727842Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3082026-07-20T22:59:56.885Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.89073893Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3092026-07-20T22:59:56.885Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.891909973Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"task":"crutest"}
3102026-07-20T22:59:56.885Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.891994553Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188}
3112026-07-20T22:59:56.885Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a81b42ee266cbdd6371f13d7b3f8dd00b79d3cab\",\n git_commit_timestamp: \"2026-07-20T22:41:44.000000000Z\",\n git_branch: \"renovate/crate-rand-vulnerability\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.89201873Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188}
3122026-07-20T22:59:56.885Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.892031529Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188}
3132026-07-20T22:59:56.886Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.892042997Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188}
3142026-07-20T22:59:56.886Z{"msg":"Crucible 03ed9c33-2fa4-4d5d-b97e-1062f2447f49 has session id: 236713eb-6275-4c3d-956c-2915d29a6ed7","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.892225926Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3152026-07-20T22:59:56.886Z{"msg":"Upstairs opts: Upstairs UUID: 03ed9c33-2fa4-4d5d-b97e-1062f2447f49, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.892259272Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3162026-07-20T22:59:56.886Z{"msg":"Crucible stats registered with UUID: 03ed9c33-2fa4-4d5d-b97e-1062f2447f49","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.89227431Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3172026-07-20T22:59:56.886Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-20T22:59:55.892283999Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3182026-07-20T22:59:56.886Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.892297007Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3192026-07-20T22:59:56.886Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.892311766Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3202026-07-20T22:59:56.886Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.892321715Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3212026-07-20T22:59:56.886Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.892436381Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"task":"crutest"}
3222026-07-20T22:59:56.886Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.89244598Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188}
3232026-07-20T22:59:56.886Z{"msg":"03ed9c33-2fa4-4d5d-b97e-1062f2447f49 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.892493634Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3242026-07-20T22:59:56.886Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.892503423Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"client":"0","":"downstairs","session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3252026-07-20T22:59:56.886Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.892514372Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"client":"1","":"downstairs","session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3262026-07-20T22:59:56.887Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.892524441Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"client":"2","":"downstairs","session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3272026-07-20T22:59:56.887Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.89253436Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3282026-07-20T22:59:56.887Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.892544438Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3292026-07-20T22:59:56.887Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.892554757Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3302026-07-20T22:59:56.890Z{"msg":"ds_connection connected from Ok(127.0.0.1:51811)","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.892636418Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3312026-07-20T22:59:56.890Z{"msg":"ds_connection connected from Ok(127.0.0.1:40489)","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.892733556Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3322026-07-20T22:59:56.891Z{"msg":"ds_connection connected from Ok(127.0.0.1:34896)","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.892803108Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3332026-07-20T22:59:56.891Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.893329256Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"client":"0","":"downstairs","session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3342026-07-20T22:59:56.891Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.893347534Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"client":"2","":"downstairs","session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3352026-07-20T22:59:56.891Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.893358903Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"client":"1","":"downstairs","session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3362026-07-20T22:59:56.891Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.893518494Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3372026-07-20T22:59:56.891Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.893532352Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3382026-07-20T22:59:56.891Z{"msg":"[0]R flush_numbers: [3995, 3995, 3995, 3995, 3995]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.893619062Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"downstairs","":"downstairs","session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3392026-07-20T22:59:56.891Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.89363394Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"downstairs","":"downstairs","session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3402026-07-20T22:59:56.891Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.893643959Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"downstairs","":"downstairs","session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3412026-07-20T22:59:56.891Z{"msg":"[1]R flush_numbers: [3995, 3995, 3995, 3995, 3995]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.893658287Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"downstairs","":"downstairs","session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3422026-07-20T22:59:56.891Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.893668216Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"downstairs","":"downstairs","session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3432026-07-20T22:59:56.891Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.893682285Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"downstairs","":"downstairs","session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3442026-07-20T22:59:56.891Z{"msg":"[2]R flush_numbers: [3995, 3995, 3995, 3995, 3995]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.893692264Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"downstairs","":"downstairs","session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3452026-07-20T22:59:56.891Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.893706212Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"downstairs","":"downstairs","session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3462026-07-20T22:59:56.891Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.893715831Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"downstairs","":"downstairs","session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3472026-07-20T22:59:56.892Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.89372582Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"downstairs","":"downstairs","session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3482026-07-20T22:59:56.892Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.893736038Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"downstairs","":"downstairs","session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3492026-07-20T22:59:56.892Z{"msg":"Next flush: 3996","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.893746017Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"downstairs","":"downstairs","session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3502026-07-20T22:59:56.892Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.893755966Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"downstairs","":"downstairs","session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3512026-07-20T22:59:56.892Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.893766455Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3522026-07-20T22:59:56.892Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.893776134Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"client":"0","":"downstairs","session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3532026-07-20T22:59:56.892Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.893786452Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"client":"1","":"downstairs","session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3542026-07-20T22:59:56.892Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.893796701Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"client":"2","":"downstairs","session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3552026-07-20T22:59:56.892Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.8938075Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3562026-07-20T22:59:56.892Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.893817889Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3572026-07-20T22:59:56.892Z{"msg":"03ed9c33-2fa4-4d5d-b97e-1062f2447f49 is now active with session: f75f48f2-16f1-421c-94c5-bd245e244622","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.893828627Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3582026-07-20T22:59:56.892Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.893838086Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"session_id":"f75f48f2-16f1-421c-94c5-bd245e244622"}
3592026-07-20T22:59:56.892Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.893847765Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188}
3602026-07-20T22:59:56.892Z{"msg":"Activated sub_volume 03ed9c33-2fa4-4d5d-b97e-1062f2447f49","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.893856464Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188}
3612026-07-20T22:59:56.892Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.893864363Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188}
3622026-07-20T22:59:56.892Z{"msg":"03ed9c33-2fa4-4d5d-b97e-1062f2447f49 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.893877382Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3632026-07-20T22:59:56.892Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.893886681Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"client":"0","":"downstairs","session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3642026-07-20T22:59:56.892Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.89389653Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"client":"1","":"downstairs","session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3652026-07-20T22:59:56.892Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.893910458Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"client":"2","":"downstairs","session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3662026-07-20T22:59:56.892Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.893969671Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3672026-07-20T22:59:56.893Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.894048312Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3682026-07-20T22:59:56.893Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.894072189Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3692026-07-20T22:59:56.893Z{"msg":"ds_connection connected from Ok(127.0.0.1:33877)","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.894083877Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3702026-07-20T22:59:56.893Z{"msg":"ds_connection connected from Ok(127.0.0.1:48200)","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.894098876Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3712026-07-20T22:59:56.893Z{"msg":"ds_connection connected from Ok(127.0.0.1:46620)","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.894109295Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3722026-07-20T22:59:56.893Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.894276375Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"client":"2","":"downstairs","session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3732026-07-20T22:59:56.893Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.894296972Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"client":"1","":"downstairs","session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3742026-07-20T22:59:56.893Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.894418658Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"client":"0","":"downstairs","session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3752026-07-20T22:59:56.893Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.894455124Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3762026-07-20T22:59:56.893Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.894472762Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3772026-07-20T22:59:56.894Z{"msg":"[0]R flush_numbers: [5316, 5316, 5316, 5316, 5316]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.894560132Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"downstairs","":"downstairs","session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3782026-07-20T22:59:56.894Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.89457486Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"downstairs","":"downstairs","session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3792026-07-20T22:59:56.894Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.894584799Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"downstairs","":"downstairs","session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3802026-07-20T22:59:56.894Z{"msg":"[1]R flush_numbers: [5316, 5316, 5316, 5316, 5316]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.894599037Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"downstairs","":"downstairs","session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3812026-07-20T22:59:56.894Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.894608726Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"downstairs","":"downstairs","session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3822026-07-20T22:59:56.894Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.894622764Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"downstairs","":"downstairs","session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3832026-07-20T22:59:56.894Z{"msg":"[2]R flush_numbers: [5316, 5316, 5316, 5316, 5316]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.894632873Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"downstairs","":"downstairs","session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3842026-07-20T22:59:56.894Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.894646751Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"downstairs","":"downstairs","session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3852026-07-20T22:59:56.894Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.89465687Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"downstairs","":"downstairs","session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3862026-07-20T22:59:56.894Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.894670609Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"downstairs","":"downstairs","session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3872026-07-20T22:59:56.894Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.894680517Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"downstairs","":"downstairs","session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3882026-07-20T22:59:56.894Z{"msg":"Next flush: 5317","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.894694146Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"downstairs","":"downstairs","session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3892026-07-20T22:59:56.894Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.894703815Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"":"downstairs","":"downstairs","session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3902026-07-20T22:59:56.894Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.894717853Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3912026-07-20T22:59:56.894Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.894726642Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"client":"0","":"downstairs","session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3922026-07-20T22:59:56.894Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.89474038Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"client":"1","":"downstairs","session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3932026-07-20T22:59:56.894Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.894749849Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"client":"2","":"downstairs","session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3942026-07-20T22:59:56.894Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.894763608Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3952026-07-20T22:59:56.894Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.894772867Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3962026-07-20T22:59:56.894Z{"msg":"03ed9c33-2fa4-4d5d-b97e-1062f2447f49 is now active with session: 236713eb-6275-4c3d-956c-2915d29a6ed7","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.894787165Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3972026-07-20T22:59:56.894Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.894795794Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"session_id":"236713eb-6275-4c3d-956c-2915d29a6ed7"}
3982026-07-20T22:59:56.894Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.894809252Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188}
3992026-07-20T22:59:56.894Z{"msg":"Activated sub_volume 03ed9c33-2fa4-4d5d-b97e-1062f2447f49","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.894823461Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188}
4002026-07-20T22:59:56.894Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.894835969Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"task":"crutest"}
4012026-07-20T22:59:56.894ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
4022026-07-20T22:59:56.894ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4032026-07-20T22:59:56.894Z{"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.895210035Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"task":"crutest"}
4042026-07-20T22:59:56.912ZRead and Verify all blocks (0..100 range:false)
4052026-07-20T22:59:56.960Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30----------------------------------------------------------------
4062026-07-20T22:59:56.960Z,"time":"2026-07-20T22:59:55.968484757Z"," Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4072026-07-20T22:59:56.960Zhostname"States::" Activew-01KY0VTGNPBVHPAV3TNY4P3W1H" Active Active
4082026-07-20T22:59:56.964Z,"JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4092026-07-20T22:59:56.964Zpid":1188,"task":"crutest" 1002 Acked Read 50}
4102026-07-20T22:59:56.964Z Done Done Done false
4112026-07-20T22:59:56.964Z{"msg":" STATES DS:0 DS:1 DS:2 TOTAL
4122026-07-20T22:59:56.964Z Sent CLIENT: Tests done. All submitted work has been ACK'd"," 0 v 0 " 0 :0 0
4132026-07-20T22:59:56.964Z," Done name 1 " 1 :" 1 3
4142026-07-20T22:59:56.964Zcrucible" Skipped 0 , 0 " 0 0
4152026-07-20T22:59:56.964Zlevel": Error 30 0 0 0 0
4162026-07-20T22:59:56.964ZLast Flush: Some(JobId(1001)) ,Some(JobId(1001)) "Some(JobId(1001)) time"
4172026-07-20T22:59:56.964Z:Downstairs last five completed:"2026-07-20T22:59:55.968522732Z" 1001, 1000"
4182026-07-20T22:59:56.964Zhostname":"Upstairs last five completed: w-01KY0VTGNPBVHPAV3TNY4P3W1H"," 1002pid 1001" 1000:
4192026-07-20T22:59:56.964Z1188,"task":"crutest"}
4202026-07-20T22:59:56.964Z----------------------------------------------------------------
4212026-07-20T22:59:56.964Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4222026-07-20T22:59:56.964ZStates: Active Active Active
4232026-07-20T22:59:56.964ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4242026-07-20T22:59:56.964Z 1002 Acked Read 50 Done Done Done false
4252026-07-20T22:59:56.964Z STATES DS:0 DS:1 DS:2 TOTAL
4262026-07-20T22:59:56.964Z Sent 0 0 0 0
4272026-07-20T22:59:56.964Z Done 1 1 1 3
4282026-07-20T22:59:56.964Z Skipped 0 0 0 0
4292026-07-20T22:59:56.964Z Error 0 0 0 0
4302026-07-20T22:59:56.964ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4312026-07-20T22:59:56.964ZDownstairs last five completed: 1001 1000
4322026-07-20T22:59:56.964ZUpstairs last five completed: 1002 1001 1000
4332026-07-20T22:59:56.964Z{"msg":"CLIENT: Up:0 ds:2 act:6","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:55.968599183Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"task":"crutest"}
4342026-07-20T23:00:00.962Z----------------------------------------------------------------
4352026-07-20T23:00:00.962Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4362026-07-20T23:00:00.962ZDownstairs last five completed: 1003 1002 1001 1000
4372026-07-20T23:00:00.962ZUpstairs last five completed: 1003 1002 1001 1000
4382026-07-20T23:00:00.966Z----------------------------------------------------------------
4392026-07-20T23:00:00.966Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4402026-07-20T23:00:00.966ZDownstairs last five completed: 1003 1002 1001 1000
4412026-07-20T23:00:00.966ZUpstairs last five completed: 1003 1002 1001 1000
4422026-07-20T23:00:00.966Z{"msg":"CLIENT: Up:0 ds:0 act:6","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:59.969961363Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"task":"crutest"}
4432026-07-20T23:00:00.966Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:59:59.970025486Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188,"task":"crutest"}
4442026-07-20T23:00:00.966Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-20T22:59:59.970044393Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1188}
4452026-07-20T23:00:00.969ZRepair part 1 passed
4462026-07-20T23:00:00.969Z
4472026-07-20T23:00:00.969ZKill the current downstairs
4482026-07-20T23:00:01.014ZDownstairs 2 stopped
4492026-07-20T23:00:01.014Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
4502026-07-20T23:00:01.017ZNow put back the original so we have a mismatch
4512026-07-20T23:00:01.017Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
4522026-07-20T23:00:01.017ZRestart downstairs with old directory
4532026-07-20T23:00:01.028Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4542026-07-20T23:00:01.040ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4552026-07-20T23:00:01.043Z 0 000-009 32 32 31 3996 3996 3995 F F F <---
4562026-07-20T23:00:01.043Z 1 010-019 32 32 31 3996 3996 3995 F F F <---
4572026-07-20T23:00:01.043Z 2 020-029 32 32 31 3996 3996 3995 F F F <---
4582026-07-20T23:00:01.043Z 3 030-039 32 32 31 3996 3996 3995 F F F <---
4592026-07-20T23:00:01.044Z 4 040-049 32 32 31 3996 3996 3995 F F F <---
4602026-07-20T23:00:01.044ZMax gen: 32, Max flush: 3996
4612026-07-20T23:00:01.044ZError: Difference in extent metadata found!
4622026-07-20T23:00:01.044Zdump test found error as expected
4632026-07-20T23:00:01.044Z
4642026-07-20T23:00:01.044Z
4652026-07-20T23:00:01.044Z/var/tmp/bins/crutest verify --range -g 33 -q --key 78/DNdfG3j2qF4kZzjqDtSDoFKLxr3veIRmQ+njh7UY= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
4662026-07-20T23:00:01.061Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.068320058Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"task":"crutest"}
4672026-07-20T23:00:01.064Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.068654618Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"task":"crutest"}
4682026-07-20T23:00:01.064Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.069540844Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"task":"crutest"}
4692026-07-20T23:00:01.064Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.069782516Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196}
4702026-07-20T23:00:01.064Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a81b42ee266cbdd6371f13d7b3f8dd00b79d3cab\",\n git_commit_timestamp: \"2026-07-20T22:41:44.000000000Z\",\n git_branch: \"renovate/crate-rand-vulnerability\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.069813882Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196}
4712026-07-20T23:00:01.065Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.069823391Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196}
4722026-07-20T23:00:01.065Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.06983538Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196}
4732026-07-20T23:00:01.065Z{"msg":"Crucible f62b6567-e425-4081-9dbe-72961f492306 has session id: dfead77e-2510-4c9a-a4e4-edc984861466","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.070214505Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
4742026-07-20T23:00:01.065Z{"msg":"Upstairs opts: Upstairs UUID: f62b6567-e425-4081-9dbe-72961f492306, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.070251511Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
4752026-07-20T23:00:01.065Z{"msg":"Crucible stats registered with UUID: f62b6567-e425-4081-9dbe-72961f492306","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.070270528Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
4762026-07-20T23:00:01.065Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-20T23:00:00.070284987Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
4772026-07-20T23:00:01.065Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.070393464Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
4782026-07-20T23:00:01.065Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.070410682Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
4792026-07-20T23:00:01.065Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.070421371Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
4802026-07-20T23:00:01.066Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.071584984Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"task":"crutest"}
4812026-07-20T23:00:01.066Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.071643467Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196}
4822026-07-20T23:00:01.066Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a81b42ee266cbdd6371f13d7b3f8dd00b79d3cab\",\n git_commit_timestamp: \"2026-07-20T22:41:44.000000000Z\",\n git_branch: \"renovate/crate-rand-vulnerability\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.071663985Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196}
4832026-07-20T23:00:01.066Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.071672424Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196}
4842026-07-20T23:00:01.066Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.071684412Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196}
4852026-07-20T23:00:01.070Z{"msg":"Crucible f62b6567-e425-4081-9dbe-72961f492306 has session id: 600a3235-f2e2-400a-b684-261cbadf9404","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.071818277Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
4862026-07-20T23:00:01.070Z{"msg":"Upstairs opts: Upstairs UUID: f62b6567-e425-4081-9dbe-72961f492306, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.071837374Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
4872026-07-20T23:00:01.070Z{"msg":"Crucible stats registered with UUID: f62b6567-e425-4081-9dbe-72961f492306","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.071867631Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
4882026-07-20T23:00:01.070Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-20T23:00:00.07187687Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
4892026-07-20T23:00:01.070Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.071886499Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
4902026-07-20T23:00:01.070Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.071897397Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
4912026-07-20T23:00:01.070Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.071907946Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
4922026-07-20T23:00:01.070Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.072019853Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"task":"crutest"}
4932026-07-20T23:00:01.070Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.072032481Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196}
4942026-07-20T23:00:01.070Z{"msg":"f62b6567-e425-4081-9dbe-72961f492306 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.072061208Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
4952026-07-20T23:00:01.070Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.072071267Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"client":"0","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
4962026-07-20T23:00:01.070Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.072081536Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"client":"1","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
4972026-07-20T23:00:01.070Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.072091965Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"client":"2","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
4982026-07-20T23:00:01.070Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.072102013Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
4992026-07-20T23:00:01.070Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.072123361Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5002026-07-20T23:00:01.071Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.072154967Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5012026-07-20T23:00:01.072Z{"msg":"ds_connection connected from Ok(127.0.0.1:43381)","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.072178414Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5022026-07-20T23:00:01.072Z{"msg":"ds_connection connected from Ok(127.0.0.1:64833)","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.072231888Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5032026-07-20T23:00:01.072Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-07-20T23:00:00.072273943Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5042026-07-20T23:00:01.072Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-07-20T23:00:00.072286782Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5052026-07-20T23:00:01.072Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-07-20T23:00:00.07229875Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5062026-07-20T23:00:01.072Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.072356253Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5072026-07-20T23:00:01.072Z{"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.072377871Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5082026-07-20T23:00:01.072Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.072880752Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"client":"1","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5092026-07-20T23:00:01.072Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.072911558Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"client":"0","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5102026-07-20T23:00:01.072Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.073014416Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5112026-07-20T23:00:01.072Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:00.073029724Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5122026-07-20T23:00:11.067Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.073058142Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5132026-07-20T23:00:11.067Z{"msg":"ds_connection connected from Ok(127.0.0.1:54291)","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.073122554Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5142026-07-20T23:00:11.073Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.074187549Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"client":"2","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5152026-07-20T23:00:11.073Z{"msg":"[0]R flush_numbers: [3996, 3996, 3996, 3996, 3996]","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.074425261Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"downstairs","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5162026-07-20T23:00:11.073Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.074441729Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"downstairs","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5172026-07-20T23:00:11.073Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.074453028Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"downstairs","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5182026-07-20T23:00:11.073Z{"msg":"[1]R flush_numbers: [3996, 3996, 3996, 3996, 3996]","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.074464937Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"downstairs","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5192026-07-20T23:00:11.073Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.074475755Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"downstairs","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5202026-07-20T23:00:11.073Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.074491884Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"downstairs","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5212026-07-20T23:00:11.073Z{"msg":"[2]R flush_numbers: [3995, 3995, 3995, 3995, 3995]","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.074503132Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"downstairs","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5222026-07-20T23:00:11.073Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.074513831Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"downstairs","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5232026-07-20T23:00:11.073Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.074529409Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"downstairs","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5242026-07-20T23:00:11.073Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.074539768Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"downstairs","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5252026-07-20T23:00:11.073Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.074550737Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"downstairs","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5262026-07-20T23:00:11.073Z{"msg":"Next flush: 3997","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.074565645Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"downstairs","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5272026-07-20T23:00:11.073Z{"msg":"extent 0 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3996, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3996, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3995, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.074580793Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5282026-07-20T23:00:11.073Z{"msg":"extent:0 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3996, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3996, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3995, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.074601481Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5292026-07-20T23:00:11.073Z{"msg":"found dest for source 0 for extent at index 0 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.074613809Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5302026-07-20T23:00:11.073Z{"msg":"extent 1 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3996, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3996, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3995, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.074633847Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5312026-07-20T23:00:11.074Z{"msg":"extent:1 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3996, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3996, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3995, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.074649375Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5322026-07-20T23:00:11.074Z{"msg":"found dest for source 0 for extent at index 1 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.074661434Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5332026-07-20T23:00:11.074Z{"msg":"extent 2 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3996, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3996, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3995, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.074676602Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5342026-07-20T23:00:11.074Z{"msg":"extent:2 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3996, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3996, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3995, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.07469631Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5352026-07-20T23:00:11.074Z{"msg":"found dest for source 0 for extent at index 2 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.074707908Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5362026-07-20T23:00:11.074Z{"msg":"extent 3 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3996, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3996, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3995, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.074728016Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5372026-07-20T23:00:11.074Z{"msg":"extent:3 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3996, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3996, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3995, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.074743414Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5382026-07-20T23:00:11.074Z{"msg":"found dest for source 0 for extent at index 3 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.074759212Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5392026-07-20T23:00:11.074Z{"msg":"extent 4 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3996, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3996, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3995, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.07477441Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5402026-07-20T23:00:11.074Z{"msg":"extent:4 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3996, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3996, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3995, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.074794208Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5412026-07-20T23:00:11.074Z{"msg":"found dest for source 0 for extent at index 4 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.074805427Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5422026-07-20T23:00:11.074Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.074823665Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"client":"0","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5432026-07-20T23:00:11.074Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.074847432Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"client":"1","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5442026-07-20T23:00:11.074Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.07486463Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"client":"2","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5452026-07-20T23:00:11.074Z{"msg":"Full repair list: {ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.074887437Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"downstairs","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5462026-07-20T23:00:11.075Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 3997, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 3997, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 3997, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 3997, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 3997, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.074989885Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"downstairs","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5472026-07-20T23:00:11.075Z{"msg":"starting reconciliation e204b999-4e5f-4b0f-b1ea-e7c40d765a6c: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.075016092Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"downstairs","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5482026-07-20T23:00:11.075Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.075096862Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"client":"2","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5492026-07-20T23:00:11.079Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.083010973Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"client":"2","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5502026-07-20T23:00:11.079Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.085112196Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"client":"2","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5512026-07-20T23:00:11.082Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.08720527Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"client":"2","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5522026-07-20T23:00:11.086Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.088928357Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"client":"2","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5532026-07-20T23:00:11.086Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.090592832Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"downstairs","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5542026-07-20T23:00:11.086Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.090623218Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"client":"0","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5552026-07-20T23:00:11.086Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.090633867Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"client":"1","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5562026-07-20T23:00:11.086Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.090648585Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"client":"2","":"downstairs","session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5572026-07-20T23:00:11.086Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.090658664Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5582026-07-20T23:00:11.086Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.090672593Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5592026-07-20T23:00:11.086Z{"msg":"f62b6567-e425-4081-9dbe-72961f492306 is now active with session: dfead77e-2510-4c9a-a4e4-edc984861466","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.090682801Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
5602026-07-20T23:00:11.086Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.09069604Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196}
5612026-07-20T23:00:11.086Z{"msg":"Activated sub_volume f62b6567-e425-4081-9dbe-72961f492306","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.090704449Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196}
5622026-07-20T23:00:11.086Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.090758382Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196}
5632026-07-20T23:00:11.086Z{"msg":"f62b6567-e425-4081-9dbe-72961f492306 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.090771721Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5642026-07-20T23:00:11.086Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.09078098Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"client":"0","":"downstairs","session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5652026-07-20T23:00:11.086Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.090795278Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"client":"1","":"downstairs","session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5662026-07-20T23:00:11.087Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.090804977Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"client":"2","":"downstairs","session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5672026-07-20T23:00:11.087Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.090848782Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5682026-07-20T23:00:11.087Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.090877149Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5692026-07-20T23:00:11.087Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.090888557Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5702026-07-20T23:00:11.087Z{"msg":"ds_connection connected from Ok(127.0.0.1:56845)","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.090899456Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5712026-07-20T23:00:11.087Z{"msg":"ds_connection connected from Ok(127.0.0.1:41059)","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.090909895Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5722026-07-20T23:00:11.087Z{"msg":"ds_connection connected from Ok(127.0.0.1:35768)","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.090920283Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5732026-07-20T23:00:11.087Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.091305858Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"client":"1","":"downstairs","session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5742026-07-20T23:00:11.087Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.091336595Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"client":"2","":"downstairs","session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5752026-07-20T23:00:11.087Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.091349543Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"client":"0","":"downstairs","session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5762026-07-20T23:00:11.087Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.091507045Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5772026-07-20T23:00:11.087Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.091520923Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5782026-07-20T23:00:11.087Z{"msg":"[0]R flush_numbers: [5317, 5317, 5317, 5317, 5317]","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.091530602Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"downstairs","":"downstairs","session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5792026-07-20T23:00:11.087Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.09154534Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"downstairs","":"downstairs","session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5802026-07-20T23:00:11.087Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.091562118Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"downstairs","":"downstairs","session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5812026-07-20T23:00:11.087Z{"msg":"[1]R flush_numbers: [5317, 5317, 5317, 5317, 5317]","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.091572577Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"downstairs","":"downstairs","session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5822026-07-20T23:00:11.087Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.091586885Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"downstairs","":"downstairs","session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5832026-07-20T23:00:11.087Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.091596724Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"downstairs","":"downstairs","session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5842026-07-20T23:00:11.087Z{"msg":"[2]R flush_numbers: [5317, 5317, 5317, 5317, 5317]","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.091607103Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"downstairs","":"downstairs","session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5852026-07-20T23:00:11.087Z{"msg":"[2]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.091621161Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"downstairs","":"downstairs","session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5862026-07-20T23:00:11.088Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.09163147Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"downstairs","":"downstairs","session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5872026-07-20T23:00:11.088Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.091645288Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"downstairs","":"downstairs","session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5882026-07-20T23:00:11.088Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.091655637Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"downstairs","":"downstairs","session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5892026-07-20T23:00:11.088Z{"msg":"Next flush: 5318","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.091669445Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"downstairs","":"downstairs","session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5902026-07-20T23:00:11.088Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.091679604Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"":"downstairs","":"downstairs","session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5912026-07-20T23:00:11.088Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.091690413Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5922026-07-20T23:00:11.088Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.091703691Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"client":"0","":"downstairs","session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5932026-07-20T23:00:11.088Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.09171332Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"client":"1","":"downstairs","session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5942026-07-20T23:00:11.088Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.091727049Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"client":"2","":"downstairs","session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5952026-07-20T23:00:11.088Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.091736958Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5962026-07-20T23:00:11.089Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.091750886Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5972026-07-20T23:00:11.089Z{"msg":"f62b6567-e425-4081-9dbe-72961f492306 is now active with session: 600a3235-f2e2-400a-b684-261cbadf9404","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.091761335Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5982026-07-20T23:00:11.089Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.091770074Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"session_id":"600a3235-f2e2-400a-b684-261cbadf9404"}
5992026-07-20T23:00:11.089Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.091779683Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196}
6002026-07-20T23:00:11.089Z{"msg":"Activated sub_volume f62b6567-e425-4081-9dbe-72961f492306","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.091788281Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196}
6012026-07-20T23:00:11.089Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.091796621Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"task":"crutest"}
6022026-07-20T23:00:11.093ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
6032026-07-20T23:00:11.093ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
6042026-07-20T23:00:11.093ZRead and Verify all blocks (0..100 range:true)
6052026-07-20T23:00:11.093Z{"msg":"flush check fired despite having jobs; resetting it","v":0,"name":"crucible","level":40,"time":"2026-07-20T23:00:10.093217804Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"session_id":"dfead77e-2510-4c9a-a4e4-edc984861466"}
6062026-07-20T23:00:11.136Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.140849298Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"task":"crutest"}
6072026-07-20T23:00:11.136Z{"msg":"Verify test completed","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.140869185Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"task":"crutest"}
6082026-07-20T23:00:11.136Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.140879634Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"task":"crutest"}
6092026-07-20T23:00:11.137Z{"msg":"CLIENT: Tests done. All submitted work has been ACK'd","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.140888383Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"task":"crutest"}
6102026-07-20T23:00:11.140Z----------------------------------------------------------------
6112026-07-20T23:00:11.140Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6122026-07-20T23:00:11.141ZStates: Active Active Active
6132026-07-20T23:00:11.141ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6142026-07-20T23:00:11.141Z 1000 Acked Read 50 Done Done Done false
6152026-07-20T23:00:11.141Z STATES DS:0 DS:1 DS:2 TOTAL
6162026-07-20T23:00:11.141Z Sent 0 0 0 0
6172026-07-20T23:00:11.141Z Done 1 1 1 3
6182026-07-20T23:00:11.141Z Skipped 0 0 0 0
6192026-07-20T23:00:11.141Z Error 0 0 0 0
6202026-07-20T23:00:11.141ZLast Flush: None None None
6212026-07-20T23:00:11.141ZDownstairs last five completed:
6222026-07-20T23:00:11.141ZUpstairs last five completed: 1000
6232026-07-20T23:00:11.141Z----------------------------------------------------------------
6242026-07-20T23:00:11.141Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6252026-07-20T23:00:11.141ZStates: Active Active Active
6262026-07-20T23:00:11.141ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6272026-07-20T23:00:11.141Z 1000 Acked Read 50 Done Done Done false
6282026-07-20T23:00:11.141Z STATES DS:0 DS:1 DS:2 TOTAL
6292026-07-20T23:00:11.141Z Sent 0 0 0 0
6302026-07-20T23:00:11.141Z Done 1 1 1 3
6312026-07-20T23:00:11.141Z Skipped 0 0 0 0
6322026-07-20T23:00:11.141Z Error 0 0 0 0
6332026-07-20T23:00:11.141ZLast Flush: None None None
6342026-07-20T23:00:11.141ZDownstairs last five completed:
6352026-07-20T23:00:11.141ZUpstairs last five completed: 1000
6362026-07-20T23:00:11.142Z{"msg":"CLIENT: Up:0 ds:2 act:6","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:10.141026097Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"task":"crutest"}
6372026-07-20T23:00:15.136Z----------------------------------------------------------------
6382026-07-20T23:00:15.136Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6392026-07-20T23:00:15.136ZDownstairs last five completed: 1001 1000
6402026-07-20T23:00:15.137ZUpstairs last five completed: 1001 1000
6412026-07-20T23:00:15.137Z----------------------------------------------------------------
6422026-07-20T23:00:15.137Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6432026-07-20T23:00:15.137ZDownstairs last five completed: 1001 1000
6442026-07-20T23:00:15.137ZUpstairs last five completed: 1001 1000
6452026-07-20T23:00:15.140Z{"msg":"CLIENT: Up:0 ds:0 act:6","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:14.142110071Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"task":"crutest"}
6462026-07-20T23:00:15.140Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-07-20T23:00:14.142185892Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196,"task":"crutest"}
6472026-07-20T23:00:15.140Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-20T23:00:14.142209279Z","hostname":"w-01KY0VTGNPBVHPAV3TNY4P3W1H","pid":1196}
6482026-07-20T23:00:15.145ZRepair part 2 passed
6492026-07-20T23:00:15.145Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
6502026-07-20T23:00:15.156ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6512026-07-20T23:00:15.159Z 0 000-009 32 32 32 3996 3996 3996 F F F
6522026-07-20T23:00:15.159Z 1 010-019 32 32 32 3996 3996 3996 F F F
6532026-07-20T23:00:15.159Z 2 020-029 32 32 32 3996 3996 3996 F F F
6542026-07-20T23:00:15.159Z 3 030-039 32 32 32 3996 3996 3996 F F F
6552026-07-20T23:00:15.159Z 4 040-049 32 32 32 3996 3996 3996 F F F
6562026-07-20T23:00:15.159ZMax gen: 32, Max flush: 3996
6572026-07-20T23:00:15.159Zdump test passed
6582026-07-20T23:00:15.159Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
6592026-07-20T23:00:15.169Z Extent 1
6602026-07-20T23:00:15.172ZGEN 32 32 32
6612026-07-20T23:00:15.172ZFLUSH_ID 3996 3996 3996
6622026-07-20T23:00:15.172ZDIRTY
6632026-07-20T23:00:15.172Z
6642026-07-20T23:00:15.172ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
6652026-07-20T23:00:15.172Z010 A A A A A A
6662026-07-20T23:00:15.172Z011 A A A A A A
6672026-07-20T23:00:15.172Z012 A A A A A A
6682026-07-20T23:00:15.172Z013 A A A A A A
6692026-07-20T23:00:15.176Z014 A A A A A A
6702026-07-20T23:00:15.176Z015 A A A A A A
6712026-07-20T23:00:15.176Z016 A A A A A A
6722026-07-20T23:00:15.176Z017 A A A A A A
6732026-07-20T23:00:15.176Z018 A A A A A A
6742026-07-20T23:00:15.176Z019 A A A A A A
6752026-07-20T23:00:15.179Zdump extent test passed
6762026-07-20T23:00:15.179Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
6772026-07-20T23:00:15.188ZExtent 2 Block in extent 0 Actual block 20
6782026-07-20T23:00:15.188Z
6792026-07-20T23:00:15.191Z DATA SHA256 VER
6802026-07-20T23:00:15.191Z------ ---------------------------------------------------------------- ---
6812026-07-20T23:00:15.191Z 0 c35d355995cec1537ee928bfec71bf608bc2048a772bb8f49c280f857a84c831 A
6822026-07-20T23:00:15.191Z 1 c35d355995cec1537ee928bfec71bf608bc2048a772bb8f49c280f857a84c831 A
6832026-07-20T23:00:15.191Z 2 c35d355995cec1537ee928bfec71bf608bc2048a772bb8f49c280f857a84c831 A
6842026-07-20T23:00:15.191Z
6852026-07-20T23:00:15.191ZNONCES 0 1 2 DIFF
6862026-07-20T23:00:15.191Z------ ------------------------ ------------------------ ------------------------ -----
6872026-07-20T23:00:15.191Z 0 e34f2750f0e411d6ffd06afc e34f2750f0e411d6ffd06afc e34f2750f0e411d6ffd06afc
6882026-07-20T23:00:15.191Z
6892026-07-20T23:00:15.191Z TAGS 0 1 2 DIFF
6902026-07-20T23:00:15.191Z------ -------------------------------- -------------------------------- -------------------------------- -----
6912026-07-20T23:00:15.191Z 0 df074e2b0fdcf3037a01df5f5bf893c4 df074e2b0fdcf3037a01df5f5bf893c4 df074e2b0fdcf3037a01df5f5bf893c4
6922026-07-20T23:00:15.191Z
6932026-07-20T23:00:15.191ZHASHES 0 1 2 DIFF
6942026-07-20T23:00:15.191Z------ ---------------- ---------------- ---------------- -----
6952026-07-20T23:00:15.191Z
6962026-07-20T23:00:15.194Zdump block test passed
6972026-07-20T23:00:15.194ZInitial upstairs tests have completed, stopping all downstairs
6982026-07-20T23:00:20.213ZCreating 7 larger downstairs regions
6992026-07-20T23:00:20.421ZStarting 7 downstairs
7002026-07-20T23:00:25.427Zdsc restarted at PID: 1212
7012026-07-20T23:00:25.528ZNow do the replace-reconcile test
7022026-07-20T23:00:25.541ZUsing 8870 for the replacement port
7032026-07-20T23:04:02.309ZCompleted test: replace-reconcile
7042026-07-20T23:04:02.313ZNow do the replace-before-active test
7052026-07-20T23:07:57.188ZCompleted test: replace-before-active
7062026-07-20T23:07:57.188ZAll tests have completed, stopping all downstairs
7072026-07-20T23:07:57.204Z
7082026-07-20T23:07:57.205ZAll Tests have passed
7092026-07-20T23:07:57.208Z11:48 Test duration
7102026-07-20T23:07:57.208Z
7112026-07-20T23:07:57.208Zreal 11:47.668988177
7122026-07-20T23:07:57.208Zuser 17:02.315323050
7132026-07-20T23:07:57.208Zsys 11.775320667
7142026-07-20T23:07:57.208Ztrap 0.426721694
7152026-07-20T23:07:57.208Ztflt 0.043028407
7162026-07-20T23:07:57.208Zdflt 0.014242967
7172026-07-20T23:07:57.208Zkflt 0.001158165
7182026-07-20T23:07:57.208Zlock 16:21:06.856543998
7192026-07-20T23:07:57.208Zslp 51:20.898098266
7202026-07-20T23:07:57.208Zlat 1:47.171544862
7212026-07-20T23:07:57.208Zstop 0.040589293
7222026-07-20T23:07:57.208Z+ echo 'test-up-2region-encrypted ends'
7232026-07-20T23:07:57.208Ztest-up-2region-encrypted ends
7242026-07-20T23:08:02.212Zprocess exited: duration 716366 ms, exit code 0
7252026-07-20T23:08:02.212Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
7262026-07-20T23:09:02.223Zexec warning: : stdout descriptor may be held open by a background process; giving up!
7272026-07-20T23:09:02.223Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
7282026-07-20T23:09:02.236Zfound 17 output files
7292026-07-20T23:09:02.236Zuploading: /tmp/test_up-build/dsc-out.txt (58484 bytes)
7302026-07-20T23:09:03.249Zuploaded: /tmp/test_up-build/dsc-out.txt
7312026-07-20T23:09:03.249Zuploading: /tmp/test_up-build/test_up_out.txt (23113887 bytes)
7322026-07-20T23:09:04.400Zuploaded: /tmp/test_up-build/test_up_out.txt
7332026-07-20T23:09:04.401Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (550974 bytes)
7342026-07-20T23:09:05.422Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
7352026-07-20T23:09:05.422Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (548345 bytes)
7362026-07-20T23:09:06.442Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
7372026-07-20T23:09:06.442Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (978704 bytes)
7382026-07-20T23:09:07.465Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
7392026-07-20T23:09:07.465Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (543042 bytes)
7402026-07-20T23:09:08.485Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
7412026-07-20T23:09:08.485Zuploading: /tmp/test_up-build/dsc/downstairs-8850.txt (1343670 bytes)
7422026-07-20T23:09:09.510Zuploaded: /tmp/test_up-build/dsc/downstairs-8850.txt
7432026-07-20T23:09:09.510Zuploading: /tmp/test_up-build/dsc/downstairs-8860.txt (47341 bytes)
7442026-07-20T23:09:10.525Zuploaded: /tmp/test_up-build/dsc/downstairs-8860.txt
7452026-07-20T23:09:10.525Zuploading: /tmp/test_up-build/dsc/downstairs-8870.txt (3580909 bytes)
7462026-07-20T23:09:11.560Zuploaded: /tmp/test_up-build/dsc/downstairs-8870.txt
7472026-07-20T23:09:11.560Zuploading: /tmp/debug/df.txt (1270 bytes)
7482026-07-20T23:09:12.570Zuploaded: /tmp/debug/df.txt
7492026-07-20T23:09:12.570Zuploading: /tmp/debug/dtrace.txt (1018799 bytes)
7502026-07-20T23:09:12.588Zupload warning: file "/tmp/debug/dtrace.txt" changed size mid upload: 1018799 -> 1156244
7512026-07-20T23:09:13.596Zuploaded: /tmp/debug/dtrace.txt
7522026-07-20T23:09:13.596Zuploading: /tmp/debug/iostat.txt (229847 bytes)
7532026-07-20T23:09:13.608Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 229847 -> 233411
7542026-07-20T23:09:14.614Zuploaded: /tmp/debug/iostat.txt
7552026-07-20T23:09:14.618Zuploading: /tmp/debug/mpstat.txt (561371 bytes)
7562026-07-20T23:09:14.636Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 561371 -> 570809
7572026-07-20T23:09:15.644Zuploaded: /tmp/debug/mpstat.txt
7582026-07-20T23:09:15.644Zuploading: /tmp/debug/paging.txt (98900 bytes)
7592026-07-20T23:09:15.654Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 98900 -> 100626
7602026-07-20T23:09:16.661Zuploaded: /tmp/debug/paging.txt
7612026-07-20T23:09:16.661Zuploading: /tmp/debug/prstat.txt (1075306 bytes)
7622026-07-20T23:09:16.678Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 1075306 -> 1093409
7632026-07-20T23:09:17.686Zuploaded: /tmp/debug/prstat.txt
7642026-07-20T23:09:17.686Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
7652026-07-20T23:09:18.696Zuploaded: /tmp/debug/psrinfo.txt
7662026-07-20T23:09:18.696Zuploading: /tmp/debug/upstairs-info.txt (168980 bytes)
7672026-07-20T23:09:19.711Zuploaded: /tmp/debug/upstairs-info.txt