01KD6T5TPHTFDF1R0VWF7HWHKT: test-up-2region-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01KD6T6EG624EQTRGDJP0F9KMD

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12025-12-24T00:13:45.771Zjob dependencies complete; ready to run (waiting for 18 m 44 s)
22025-12-24T00:14:48.031Zjob assigned to worker 01KD6V8WYBDEH5N3JQ2EV305S3 [factory aws, i-0c69578f2c5d82494] (queued for 1 m 2 s)
32025-12-24T00:14:55.948Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42025-12-24T00:14:57.389Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52025-12-24T00:14:57.389Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62025-12-24T00:14:58.759Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72025-12-24T00:14:58.760Zdownloading input: /input/build/work/bins/crutest.gz
82025-12-24T00:15:00.197Zdownloaded input: /input/build/work/bins/crutest.gz
92025-12-24T00:15:00.197Zdownloading input: /input/build/work/bins/dsc.gz
102025-12-24T00:15:00.939Zdownloaded input: /input/build/work/bins/dsc.gz
112025-12-24T00:15:00.942Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122025-12-24T00:15:00.957Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132025-12-24T00:15:00.957Zdownloading input: /input/build/work/scripts/test_ds.sh
142025-12-24T00:15:00.977Zdownloaded input: /input/build/work/scripts/test_ds.sh
152025-12-24T00:15:00.978Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162025-12-24T00:15:01.008Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172025-12-24T00:15:01.009Zdownloading input: /input/build/work/scripts/test_repair.sh
182025-12-24T00:15:01.038Zdownloaded input: /input/build/work/scripts/test_repair.sh
192025-12-24T00:15:01.038Zdownloading input: /input/build/work/scripts/test_replay.sh
202025-12-24T00:15:01.074Zdownloaded input: /input/build/work/scripts/test_replay.sh
212025-12-24T00:15:01.074Zdownloading input: /input/build/work/scripts/test_up.sh
222025-12-24T00:15:01.104Zdownloaded input: /input/build/work/scripts/test_up.sh
232025-12-24T00:15:01.105Zdownloading input: /input/build/work/scripts/upstairs_info.d
242025-12-24T00:15:01.147Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252025-12-24T00:15:01.158Zdownloading input: /input/build/tmp/cargo-test-out.log
262025-12-24T00:15:02.202Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272025-12-24T00:15:02.202Zstarting task 0: "setup"
282025-12-24T00:15:02.212Z++ uname -s
292025-12-24T00:15:02.218Z+ kern=SunOS
302025-12-24T00:15:02.218Z+ build_user=build
312025-12-24T00:15:02.218Z+ build_uid=12345
322025-12-24T00:15:02.219Z+ work_dir=/work
332025-12-24T00:15:02.219Z+ input_dir=/input
342025-12-24T00:15:02.219Z+ [[ 0 == 12345 ]]
352025-12-24T00:15:02.219Z+ case "$kern" in
362025-12-24T00:15:02.219Z+ groupadd -g 12345 build
372025-12-24T00:15:02.220Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382025-12-24T00:15:04.221Z+ zfs create -o mountpoint=/work rpool/work
392025-12-24T00:15:04.567Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402025-12-24T00:15:04.572Z+ home_fs=zfs
412025-12-24T00:15:04.572Z+ [[ zfs == autofs ]]
422025-12-24T00:15:04.572Z+ mkdir -p /home/build
432025-12-24T00:15:04.572Z+ chown build:build /home/build /work
442025-12-24T00:15:06.573Z+ chmod 0700 /home/build /work
452025-12-24T00:15:06.577Zprocess exited: duration 4375 ms, exit code 0
 
462025-12-24T00:15:06.584Zstarting task 1: "authentication"
472025-12-24T00:15:06.605Zprocess exited: duration 21 ms, exit code 0
 
482025-12-24T00:15:06.613Zstarting task 2: "build"
492025-12-24T00:15:06.617Z+ banner cores
502025-12-24T00:15:06.621Z
512025-12-24T00:15:06.621Z #### #### ##### ###### ####
522025-12-24T00:15:06.621Z # # # # # # # #
532025-12-24T00:15:06.621Z # # # # # ##### ####
542025-12-24T00:15:06.621Z # # # ##### # #
552025-12-24T00:15:06.621Z # # # # # # # # #
562025-12-24T00:15:06.621Z #### #### # # ###### ####
572025-12-24T00:15:06.621Z
582025-12-24T00:15:06.624Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
592025-12-24T00:15:06.630Z+ banner unpack
602025-12-24T00:15:06.635Z
612025-12-24T00:15:06.635Z # # # # ##### ## #### # #
622025-12-24T00:15:06.635Z # # ## # # # # # # # # #
632025-12-24T00:15:06.635Z # # # # # # # # # # ####
642025-12-24T00:15:06.635Z # # # # # ##### ###### # # #
652025-12-24T00:15:06.635Z # # # ## # # # # # # #
662025-12-24T00:15:06.635Z #### # # # # # #### # #
672025-12-24T00:15:06.635Z
682025-12-24T00:15:06.635Z+ mkdir -p /var/tmp/bins
692025-12-24T00:15:06.639Z+ for t in "$input/bins/"*.gz
702025-12-24T00:15:06.639Z++ basename /input/build/work/bins/crucible-downstairs.gz
712025-12-24T00:15:06.639Z+ b=crucible-downstairs.gz
722025-12-24T00:15:06.639Z+ b=crucible-downstairs
732025-12-24T00:15:06.639Z+ gunzip
742025-12-24T00:15:09.582Z+ chmod +x /var/tmp/bins/crucible-downstairs
752025-12-24T00:15:09.585Z+ for t in "$input/bins/"*.gz
762025-12-24T00:15:09.589Z++ basename /input/build/work/bins/crucible-hammer.gz
772025-12-24T00:15:09.589Z+ b=crucible-hammer.gz
782025-12-24T00:15:09.589Z+ b=crucible-hammer
792025-12-24T00:15:09.589Z+ gunzip
802025-12-24T00:15:12.403Z+ chmod +x /var/tmp/bins/crucible-hammer
812025-12-24T00:15:12.408Z+ for t in "$input/bins/"*.gz
822025-12-24T00:15:12.408Z++ basename /input/build/work/bins/crutest.gz
832025-12-24T00:15:12.408Z+ b=crutest.gz
842025-12-24T00:15:12.408Z+ b=crutest
852025-12-24T00:15:12.408Z+ gunzip
862025-12-24T00:15:15.344Z+ chmod +x /var/tmp/bins/crutest
872025-12-24T00:15:15.347Z+ for t in "$input/bins/"*.gz
882025-12-24T00:15:15.351Z++ basename /input/build/work/bins/dsc.gz
892025-12-24T00:15:15.351Z+ b=dsc.gz
902025-12-24T00:15:15.351Z+ b=dsc
912025-12-24T00:15:15.351Z+ gunzip
922025-12-24T00:15:16.824Z+ chmod +x /var/tmp/bins/dsc
932025-12-24T00:15:16.828Z+ export BINDIR=/var/tmp/bins
942025-12-24T00:15:16.828Z+ BINDIR=/var/tmp/bins
952025-12-24T00:15:16.828Z+ jobpid=1157
962025-12-24T00:15:16.828Z+ echo 'Setup debug logging'
972025-12-24T00:15:16.828ZSetup debug logging
982025-12-24T00:15:16.828Z+ mkdir /tmp/debug
992025-12-24T00:15:16.828Z+ sleep 3600
1002025-12-24T00:15:16.831Z+ psrinfo -v
1012025-12-24T00:15:16.835Z+ df -h
1022025-12-24T00:15:16.841Z+ prstat -d d -mLc 1
1032025-12-24T00:15:16.844Z+ iostat -T d -xn 1
1042025-12-24T00:15:16.844Z+ mpstat -T d 1
1052025-12-24T00:15:16.844Z+ vmstat -T d -p 1
1062025-12-24T00:15:16.844Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1072025-12-24T00:15:16.849Z+ banner test_up_2r_encrypted
1082025-12-24T00:15:16.849Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1092025-12-24T00:15:16.849Z #####
1102025-12-24T00:15:16.849Z ##### ###### #### ##### # # ##### # # #####
1112025-12-24T00:15:16.849Z # # # # # # # # # # #
1122025-12-24T00:15:16.849Z # ##### #### # # # # # ##### # #
1132025-12-24T00:15:16.849Z # # # # # # ##### # #####
1142025-12-24T00:15:16.849Z # # # # # # # # # # #
1152025-12-24T00:15:16.849Z # ###### #### # ####### #### # ####### ####### # #
1162025-12-24T00:15:16.849Z
1172025-12-24T00:15:16.849Z+ ptime -m bash /input/build/work/scripts/test_up.sh -r 2 -N encrypted
1182025-12-24T00:15:16.860Z/input/build/work
1192025-12-24T00:15:16.872ZUsing 2 region sets
1202025-12-24T00:15:16.872ZTurn off color for downstairs dump
1212025-12-24T00:15:16.887ZUpstairs using key: vCBz495xm8h3swf+pJ4olsmPRhCDINMhtV6oSiYw4s8=
1222025-12-24T00:15:16.891Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1232025-12-24T00:15:16.891ZCreating 6 downstairs regions
1242025-12-24T00:15:17.241ZStarting 6 downstairs
1252025-12-24T00:15:17.244Zdsc started at PID: 1205
1262025-12-24T00:15:22.585ZDisable automatic restart on all downstairs
1272025-12-24T00:15:22.637Z
1282025-12-24T00:15:22.637ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1292025-12-24T00:15:22.641ZRunning test: span
1302025-12-24T00:15:26.834ZCompleted test: span
1312025-12-24T00:15:31.841ZRunning test: big
1322025-12-24T00:15:36.421ZCompleted test: big
1332025-12-24T00:15:41.429ZRunning test: dep
1342025-12-24T00:16:14.765ZCompleted test: dep
1352025-12-24T00:16:19.772ZRunning test: balloon
1362025-12-24T00:20:32.659ZCompleted test: balloon
1372025-12-24T00:20:37.667ZRunning test: deactivate
1382025-12-24T00:20:42.331ZCompleted test: deactivate
1392025-12-24T00:20:47.339ZRunning hammer
1402025-12-24T00:21:00.148ZRun repair tests
1412025-12-24T00:21:00.152Z/var/tmp/bins/crutest fill -g 31 -q --key vCBz495xm8h3swf+pJ4olsmPRhCDINMhtV6oSiYw4s8= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1422025-12-24T00:21:00.215Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.015530374Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"task":"crutest"}
1432025-12-24T00:21:00.219Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.016173245Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"task":"crutest"}
1442025-12-24T00:21:00.219Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.018139691Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"task":"crutest"}
1452025-12-24T00:21:00.219Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.01842538Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248}
1462025-12-24T00:21:00.219Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4c4b956995d7a3110f080eebc9768bd86623f0f1\",\n git_commit_timestamp: \"2025-12-23T23:54:01.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.018479894Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248}
1472025-12-24T00:21:00.219Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.018511891Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248}
1482025-12-24T00:21:00.219Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.018541058Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248}
1492025-12-24T00:21:00.219Z{"msg":"Crucible a62f4ad6-b3fd-42b0-94cf-090a87affb81 has session id: f58b307e-594c-4bf8-8875-f20894d21754","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.019097408Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1502025-12-24T00:21:00.219Z{"msg":"Upstairs opts: Upstairs UUID: a62f4ad6-b3fd-42b0-94cf-090a87affb81, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.019152762Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1512025-12-24T00:21:00.219Z{"msg":"Crucible stats registered with UUID: a62f4ad6-b3fd-42b0-94cf-090a87affb81","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.019193387Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1522025-12-24T00:21:00.219Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-12-24T00:20:59.019213385Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1532025-12-24T00:21:00.219Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.019389886Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1542025-12-24T00:21:00.219Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.01944149Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1552025-12-24T00:21:00.219Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.019462988Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1562025-12-24T00:21:00.230Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.021411007Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"task":"crutest"}
1572025-12-24T00:21:00.230Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.021498467Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248}
1582025-12-24T00:21:00.230Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4c4b956995d7a3110f080eebc9768bd86623f0f1\",\n git_commit_timestamp: \"2025-12-23T23:54:01.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.021546302Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248}
1592025-12-24T00:21:00.230Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.02156917Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248}
1602025-12-24T00:21:00.230Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.021591097Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248}
1612025-12-24T00:21:00.230Z{"msg":"Crucible a62f4ad6-b3fd-42b0-94cf-090a87affb81 has session id: 42b456f6-e4a0-4a8b-8820-f04c08ec3e54","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.022007232Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
1622025-12-24T00:21:00.230Z{"msg":"Upstairs opts: Upstairs UUID: a62f4ad6-b3fd-42b0-94cf-090a87affb81, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.022082334Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
1632025-12-24T00:21:00.231Z{"msg":"Crucible stats registered with UUID: a62f4ad6-b3fd-42b0-94cf-090a87affb81","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.022136258Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
1642025-12-24T00:21:00.231Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-12-24T00:20:59.022175114Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
1652025-12-24T00:21:00.231Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.02220953Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
1662025-12-24T00:21:00.231Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.022246276Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
1672025-12-24T00:21:00.231Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.022285712Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
1682025-12-24T00:21:00.231Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.022757241Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"task":"crutest"}
1692025-12-24T00:21:00.231Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.022811475Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248}
1702025-12-24T00:21:00.231Z{"msg":"a62f4ad6-b3fd-42b0-94cf-090a87affb81 active request set","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.022844731Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1712025-12-24T00:21:00.231Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.022875298Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"client":"0","":"downstairs","session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1722025-12-24T00:21:00.231Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.022923243Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"client":"1","":"downstairs","session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1732025-12-24T00:21:00.231Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.022975437Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"client":"2","":"downstairs","session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1742025-12-24T00:21:00.231Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.023010683Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1752025-12-24T00:21:00.231Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.023051509Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1762025-12-24T00:21:00.231Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.023088965Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1772025-12-24T00:21:00.231Z{"msg":"ds_connection connected from Ok(127.0.0.1:34365)","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.023123011Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1782025-12-24T00:21:00.231Z{"msg":"ds_connection connected from Ok(127.0.0.1:40552)","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.023166257Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1792025-12-24T00:21:00.231Z{"msg":"ds_connection connected from Ok(127.0.0.1:43985)","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.023197213Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1802025-12-24T00:21:00.231Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.024316412Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"client":"1","":"downstairs","session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1812025-12-24T00:21:00.231Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.024364697Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"client":"2","":"downstairs","session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1822025-12-24T00:21:00.231Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.024399873Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"client":"0","":"downstairs","session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1832025-12-24T00:21:00.231Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.024787041Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1842025-12-24T00:21:00.231Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.024829796Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1852025-12-24T00:21:00.231Z{"msg":"[0]R flush_numbers: [3999, 3999, 3999, 3999, 3999]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.024858643Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"downstairs","":"downstairs","session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1862025-12-24T00:21:00.231Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.02489217Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"downstairs","":"downstairs","session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1872025-12-24T00:21:00.231Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.024923116Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"downstairs","":"downstairs","session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1882025-12-24T00:21:00.232Z{"msg":"[1]R flush_numbers: [3999, 3999, 3999, 3999, 3999]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.024959502Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"downstairs","":"downstairs","session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1892025-12-24T00:21:00.232Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.024997658Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"downstairs","":"downstairs","session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1902025-12-24T00:21:00.232Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.025037234Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"downstairs","":"downstairs","session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1912025-12-24T00:21:00.232Z{"msg":"[2]R flush_numbers: [3999, 3999, 3999, 3999, 3999]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.02506907Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"downstairs","":"downstairs","session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1922025-12-24T00:21:00.232Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.025101127Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"downstairs","":"downstairs","session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1932025-12-24T00:21:00.232Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.025131553Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"downstairs","":"downstairs","session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1942025-12-24T00:21:00.238Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.025191357Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"downstairs","":"downstairs","session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1952025-12-24T00:21:00.238Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.025223023Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"downstairs","":"downstairs","session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1962025-12-24T00:21:00.239Z{"msg":"Next flush: 4000","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.025251051Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"downstairs","":"downstairs","session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1972025-12-24T00:21:00.239Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.025291026Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"downstairs","":"downstairs","session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1982025-12-24T00:21:00.239Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.025491914Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
1992025-12-24T00:21:00.239Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.025538869Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"client":"0","":"downstairs","session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
2002025-12-24T00:21:00.239Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.025571286Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"client":"1","":"downstairs","session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
2012025-12-24T00:21:00.239Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.025617391Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"client":"2","":"downstairs","session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
2022025-12-24T00:21:00.239Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.025707131Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
2032025-12-24T00:21:00.239Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.025746737Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
2042025-12-24T00:21:00.239Z{"msg":"a62f4ad6-b3fd-42b0-94cf-090a87affb81 is now active with session: f58b307e-594c-4bf8-8875-f20894d21754","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.025794102Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
2052025-12-24T00:21:00.239Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.025823509Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"session_id":"f58b307e-594c-4bf8-8875-f20894d21754"}
2062025-12-24T00:21:00.239Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.025842616Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248}
2072025-12-24T00:21:00.239Z{"msg":"Activated sub_volume a62f4ad6-b3fd-42b0-94cf-090a87affb81","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.025874123Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248}
2082025-12-24T00:21:00.239Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.025890671Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248}
2092025-12-24T00:21:00.239Z{"msg":"a62f4ad6-b3fd-42b0-94cf-090a87affb81 active request set","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.02590485Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2102025-12-24T00:21:00.239Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.025929807Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"client":"0","":"downstairs","session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2112025-12-24T00:21:00.239Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.025978872Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"client":"1","":"downstairs","session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2122025-12-24T00:21:00.239Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.026052853Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"client":"2","":"downstairs","session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2132025-12-24T00:21:00.239Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.026092129Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2142025-12-24T00:21:00.239Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.026119246Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2152025-12-24T00:21:00.239Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.026147523Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2162025-12-24T00:21:00.239Z{"msg":"ds_connection connected from Ok(127.0.0.1:34866)","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.026166141Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2172025-12-24T00:21:00.239Z{"msg":"ds_connection connected from Ok(127.0.0.1:54297)","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.026194598Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2182025-12-24T00:21:00.239Z{"msg":"ds_connection connected from Ok(127.0.0.1:37465)","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.026214596Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2192025-12-24T00:21:00.239Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.02645699Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"client":"1","":"downstairs","session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2202025-12-24T00:21:00.239Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.026538201Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"client":"2","":"downstairs","session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2212025-12-24T00:21:00.239Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.026585836Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"client":"0","":"downstairs","session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2222025-12-24T00:21:00.239Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.026808912Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2232025-12-24T00:21:00.239Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.026846527Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2242025-12-24T00:21:00.239Z{"msg":"[0]R flush_numbers: [5316, 5318, 5318, 5320, 5317]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.026863776Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"downstairs","":"downstairs","session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2252025-12-24T00:21:00.239Z{"msg":"[0]R generation: [13, 14, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.026889633Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"downstairs","":"downstairs","session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2262025-12-24T00:21:00.239Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.026907501Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"downstairs","":"downstairs","session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2272025-12-24T00:21:00.239Z{"msg":"[1]R flush_numbers: [5316, 5318, 5318, 5320, 5317]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.026931258Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"downstairs","":"downstairs","session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2282025-12-24T00:21:00.239Z{"msg":"[1]R generation: [13, 14, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.026948816Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"downstairs","":"downstairs","session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2292025-12-24T00:21:00.239Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.026966285Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"downstairs","":"downstairs","session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2302025-12-24T00:21:00.239Z{"msg":"[2]R flush_numbers: [5316, 5318, 5318, 5320, 5317]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.026989962Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"downstairs","":"downstairs","session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2312025-12-24T00:21:00.240Z{"msg":"[2]R generation: [13, 14, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.02700706Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"downstairs","":"downstairs","session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2322025-12-24T00:21:00.240Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.027032377Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"downstairs","":"downstairs","session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2332025-12-24T00:21:00.240Z{"msg":"Max found gen is 15","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.027061344Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"downstairs","":"downstairs","session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2342025-12-24T00:21:00.240Z{"msg":"Generation requested: 31 >= found:15","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.027080462Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"downstairs","":"downstairs","session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2352025-12-24T00:21:00.240Z{"msg":"Next flush: 5321","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.02710372Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"downstairs","":"downstairs","session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2362025-12-24T00:21:00.240Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.027121018Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"":"downstairs","":"downstairs","session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2372025-12-24T00:21:00.240ZWait for a query_work_queue command to finish before sending IO
2382025-12-24T00:21:00.240Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.027150574Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2392025-12-24T00:21:00.240Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.027172602Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"client":"0","":"downstairs","session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2402025-12-24T00:21:00.240Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.02718928Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"client":"1","":"downstairs","session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2412025-12-24T00:21:00.240Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.027206279Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"client":"2","":"downstairs","session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2422025-12-24T00:21:00.240Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.027229436Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2432025-12-24T00:21:00.240Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.027245744Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2442025-12-24T00:21:00.240Z{"msg":"a62f4ad6-b3fd-42b0-94cf-090a87affb81 is now active with session: 42b456f6-e4a0-4a8b-8820-f04c08ec3e54","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.027269712Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2452025-12-24T00:21:00.240Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.02728547Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248,"session_id":"42b456f6-e4a0-4a8b-8820-f04c08ec3e54"}
2462025-12-24T00:21:00.240Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.027307708Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248}
2472025-12-24T00:21:00.240Z{"msg":"Activated sub_volume a62f4ad6-b3fd-42b0-94cf-090a87affb81","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:20:59.027321786Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1248}
2482025-12-24T00:21:00.240ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
2492025-12-24T00:21:00.240ZFill test
2502025-12-24T00:21:00.268ZRead and Verify all blocks (0..100 range:false)
2512025-12-24T00:21:00.348ZWrote out file "/var/tmp/test_up-build/verify_file"
2522025-12-24T00:21:00.348ZCLIENT: Tests done. All submitted work has been ACK'd
2532025-12-24T00:21:00.352Z----------------------------------------------------------------
2542025-12-24T00:21:00.352Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2552025-12-24T00:21:00.352ZStates: Active Active Active
2562025-12-24T00:21:00.352ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2572025-12-24T00:21:00.352Z 1002 Acked Read 50 Done Done Done false
2582025-12-24T00:21:00.352Z STATES DS:0 DS:1 DS:2 TOTAL
2592025-12-24T00:21:00.352Z Sent 0 0 0 0
2602025-12-24T00:21:00.352Z Done 1 1 1 3
2612025-12-24T00:21:00.352Z Skipped 0 0 0 0
2622025-12-24T00:21:00.352Z Error 0 0 0 0
2632025-12-24T00:21:00.352ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2642025-12-24T00:21:00.352ZDownstairs last five completed: 1001 1000
2652025-12-24T00:21:00.352ZUpstairs last five completed: 1002 1001 1000
2662025-12-24T00:21:00.352Z----------------------------------------------------------------
2672025-12-24T00:21:00.353Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2682025-12-24T00:21:00.353ZStates: Active Active Active
2692025-12-24T00:21:00.353ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2702025-12-24T00:21:00.353Z 1002 Acked Read 50 Done Done Done false
2712025-12-24T00:21:00.353Z STATES DS:0 DS:1 DS:2 TOTAL
2722025-12-24T00:21:00.353Z Sent 0 0 0 0
2732025-12-24T00:21:00.353Z Done 1 1 1 3
2742025-12-24T00:21:00.353Z Skipped 0 0 0 0
2752025-12-24T00:21:00.353Z Error 0 0 0 0
2762025-12-24T00:21:00.353ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2772025-12-24T00:21:00.353ZDownstairs last five completed: 1001 1000
2782025-12-24T00:21:00.353ZUpstairs last five completed: 1002 1001 1000
2792025-12-24T00:21:00.353ZCLIENT: Up:0 ds:2 act:6
2802025-12-24T00:21:04.349Z----------------------------------------------------------------
2812025-12-24T00:21:04.349Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2822025-12-24T00:21:04.349ZDownstairs last five completed: 1003 1002 1001 1000
2832025-12-24T00:21:04.349ZUpstairs last five completed: 1003 1002 1001 1000
2842025-12-24T00:21:04.352Z----------------------------------------------------------------
2852025-12-24T00:21:04.352Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2862025-12-24T00:21:04.352ZDownstairs last five completed: 1003 1002 1001 1000
2872025-12-24T00:21:04.352ZUpstairs last five completed: 1003 1002 1001 1000
2882025-12-24T00:21:04.352ZCLIENT: Up:0 ds:0 act:6
2892025-12-24T00:21:04.352ZCLIENT: All crucible jobs finished, exiting program
2902025-12-24T00:21:04.360ZRepair setup passed
2912025-12-24T00:21:04.364ZCopy the region for /var/tmp/test_up-build/8830
2922025-12-24T00:21:04.368Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2932025-12-24T00:21:04.368Z/var/tmp/bins/crutest fill -g 32 -q --key vCBz495xm8h3swf+pJ4olsmPRhCDINMhtV6oSiYw4s8= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2942025-12-24T00:21:04.426Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.227238065Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"task":"crutest"}
2952025-12-24T00:21:04.430Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.227773087Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"task":"crutest"}
2962025-12-24T00:21:04.430Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.229550944Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"task":"crutest"}
2972025-12-24T00:21:04.430Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.229827244Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254}
2982025-12-24T00:21:04.430Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4c4b956995d7a3110f080eebc9768bd86623f0f1\",\n git_commit_timestamp: \"2025-12-23T23:54:01.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.229916335Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254}
2992025-12-24T00:21:04.430Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.229974448Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254}
3002025-12-24T00:21:04.430Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.230036482Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254}
3012025-12-24T00:21:04.430Z{"msg":"Crucible 1f16c9a6-2532-4686-8173-16d9b9d6bcca has session id: be2aa5fe-d4c4-4253-b814-f531f8c0556e","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.230485053Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3022025-12-24T00:21:04.430Z{"msg":"Upstairs opts: Upstairs UUID: 1f16c9a6-2532-4686-8173-16d9b9d6bcca, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.230546116Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3032025-12-24T00:21:04.430Z{"msg":"Crucible stats registered with UUID: 1f16c9a6-2532-4686-8173-16d9b9d6bcca","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.230588622Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3042025-12-24T00:21:04.430Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-12-24T00:21:03.230625338Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3052025-12-24T00:21:04.430Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.230773602Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3062025-12-24T00:21:04.430Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.230827606Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3072025-12-24T00:21:04.430Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.230847584Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3082025-12-24T00:21:04.434Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.232697133Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"task":"crutest"}
3092025-12-24T00:21:04.434Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.232783854Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254}
3102025-12-24T00:21:04.434Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4c4b956995d7a3110f080eebc9768bd86623f0f1\",\n git_commit_timestamp: \"2025-12-23T23:54:01.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.232831799Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254}
3112025-12-24T00:21:04.434Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.232854156Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254}
3122025-12-24T00:21:04.434Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.232876434Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254}
3132025-12-24T00:21:04.434Z{"msg":"Crucible 1f16c9a6-2532-4686-8173-16d9b9d6bcca has session id: ed26f1f2-ddd6-406c-9274-4af2c7d360e4","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.233242874Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3142025-12-24T00:21:04.434Z{"msg":"Upstairs opts: Upstairs UUID: 1f16c9a6-2532-4686-8173-16d9b9d6bcca, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.233316236Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3152025-12-24T00:21:04.434Z{"msg":"Crucible stats registered with UUID: 1f16c9a6-2532-4686-8173-16d9b9d6bcca","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.233359441Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3162025-12-24T00:21:04.434Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-12-24T00:21:03.233401757Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3172025-12-24T00:21:04.434Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.233424084Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3182025-12-24T00:21:04.434Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.23345886Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3192025-12-24T00:21:04.434Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.233478878Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3202025-12-24T00:21:04.434Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.233948887Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"task":"crutest"}
3212025-12-24T00:21:04.434Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.233992523Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254}
3222025-12-24T00:21:04.434Z{"msg":"1f16c9a6-2532-4686-8173-16d9b9d6bcca active request set","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.234037438Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3232025-12-24T00:21:04.434Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.234078224Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"client":"0","":"downstairs","session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3242025-12-24T00:21:04.434Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.234126238Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"client":"1","":"downstairs","session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3252025-12-24T00:21:04.434Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.234171513Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"client":"2","":"downstairs","session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3262025-12-24T00:21:04.434Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.234218208Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3272025-12-24T00:21:04.434Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.234263313Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3282025-12-24T00:21:04.434Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.234307248Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3292025-12-24T00:21:04.434Z{"msg":"ds_connection connected from Ok(127.0.0.1:42443)","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.234351424Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3302025-12-24T00:21:04.434Z{"msg":"ds_connection connected from Ok(127.0.0.1:41751)","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.23448174Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3312025-12-24T00:21:04.434Z{"msg":"ds_connection connected from Ok(127.0.0.1:56760)","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.23457142Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3322025-12-24T00:21:04.440Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.235625156Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"client":"1","":"downstairs","session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3332025-12-24T00:21:04.440Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.235711136Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"client":"2","":"downstairs","session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3342025-12-24T00:21:04.440Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.235790278Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"client":"0","":"downstairs","session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3352025-12-24T00:21:04.440Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.236066178Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3362025-12-24T00:21:04.440Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.236105744Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3372025-12-24T00:21:04.440Z{"msg":"[0]R flush_numbers: [4000, 4000, 4000, 4000, 4000]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.236126341Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"downstairs","":"downstairs","session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3382025-12-24T00:21:04.440Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.236157898Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"downstairs","":"downstairs","session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3392025-12-24T00:21:04.440Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.236201613Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"downstairs","":"downstairs","session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3402025-12-24T00:21:04.440Z{"msg":"[1]R flush_numbers: [4000, 4000, 4000, 4000, 4000]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.236221941Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"downstairs","":"downstairs","session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3412025-12-24T00:21:04.440Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.236247078Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"downstairs","":"downstairs","session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3422025-12-24T00:21:04.440Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.236264726Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"downstairs","":"downstairs","session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3432025-12-24T00:21:04.441Z{"msg":"[2]R flush_numbers: [4000, 4000, 4000, 4000, 4000]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.236293813Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"downstairs","":"downstairs","session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3442025-12-24T00:21:04.441Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.236310881Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"downstairs","":"downstairs","session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3452025-12-24T00:21:04.441Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.236340688Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"downstairs","":"downstairs","session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3462025-12-24T00:21:04.441Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.236357076Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"downstairs","":"downstairs","session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3472025-12-24T00:21:04.441Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.236386043Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"downstairs","":"downstairs","session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3482025-12-24T00:21:04.441Z{"msg":"Next flush: 4001","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.236402321Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"downstairs","":"downstairs","session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3492025-12-24T00:21:04.441Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.236425769Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"downstairs","":"downstairs","session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3502025-12-24T00:21:04.441Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.236442947Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3512025-12-24T00:21:04.441Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.236457915Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"client":"0","":"downstairs","session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3522025-12-24T00:21:04.441Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.236475444Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"client":"1","":"downstairs","session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3532025-12-24T00:21:04.441Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.236492082Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"client":"2","":"downstairs","session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3542025-12-24T00:21:04.441Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.23650876Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3552025-12-24T00:21:04.441Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.236536507Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3562025-12-24T00:21:04.441Z{"msg":"1f16c9a6-2532-4686-8173-16d9b9d6bcca is now active with session: be2aa5fe-d4c4-4253-b814-f531f8c0556e","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.236565454Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3572025-12-24T00:21:04.441Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.236609869Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"session_id":"be2aa5fe-d4c4-4253-b814-f531f8c0556e"}
3582025-12-24T00:21:04.441Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.236645675Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254}
3592025-12-24T00:21:04.441Z{"msg":"Activated sub_volume 1f16c9a6-2532-4686-8173-16d9b9d6bcca","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.236686431Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254}
3602025-12-24T00:21:04.441Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.236712638Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254}
3612025-12-24T00:21:04.441Z{"msg":"1f16c9a6-2532-4686-8173-16d9b9d6bcca active request set","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.236870421Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3622025-12-24T00:21:04.441Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.236912326Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"client":"0","":"downstairs","session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3632025-12-24T00:21:04.441Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.236947592Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"client":"1","":"downstairs","session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3642025-12-24T00:21:04.441Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.236979249Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"client":"2","":"downstairs","session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3652025-12-24T00:21:04.441Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.237037863Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3662025-12-24T00:21:04.441Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.237079038Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3672025-12-24T00:21:04.441Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.237129113Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3682025-12-24T00:21:04.441Z{"msg":"ds_connection connected from Ok(127.0.0.1:43381)","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.237170348Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3692025-12-24T00:21:04.441Z{"msg":"ds_connection connected from Ok(127.0.0.1:40739)","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.237201615Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3702025-12-24T00:21:04.441Z{"msg":"ds_connection connected from Ok(127.0.0.1:47702)","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.237233601Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3712025-12-24T00:21:04.441Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.23761129Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"client":"0","":"downstairs","session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3722025-12-24T00:21:04.441Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.237679743Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"client":"1","":"downstairs","session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3732025-12-24T00:21:04.441Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.237905509Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"client":"2","":"downstairs","session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3742025-12-24T00:21:04.441Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.23808153Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3752025-12-24T00:21:04.441Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.238124175Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3762025-12-24T00:21:04.441Z{"msg":"[0]R flush_numbers: [5321, 5321, 5321, 5321, 5321]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.238231263Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"downstairs","":"downstairs","session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3772025-12-24T00:21:04.441Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.23826285Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"downstairs","":"downstairs","session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3782025-12-24T00:21:04.441Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.238287507Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"downstairs","":"downstairs","session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3792025-12-24T00:21:04.441Z{"msg":"[1]R flush_numbers: [5321, 5321, 5321, 5321, 5321]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.238320473Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"downstairs","":"downstairs","session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3802025-12-24T00:21:04.441Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.23835487Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"downstairs","":"downstairs","session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3812025-12-24T00:21:04.442Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.238393316Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","Wait for a query_work_queue command to finish before sending IO
3822025-12-24T00:21:04.442Zpid":1254,"":"downstairs","":"downstairs","session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3832025-12-24T00:21:04.442Z{"msg":"[2]R flush_numbers: [5321, 5321, 5321, 5321, 5321]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.238437681Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"downstairs","":"downstairs","session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3842025-12-24T00:21:04.442Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.238473877Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"downstairs","":"downstairs","session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3852025-12-24T00:21:04.442Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.238508793Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"downstairs","":"downstairs","session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3862025-12-24T00:21:04.442Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.238547669Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"downstairs","":"downstairs","session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3872025-12-24T00:21:04.442Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.238585195Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"downstairs","":"downstairs","session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3882025-12-24T00:21:04.442Z{"msg":"Next flush: 5322","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.238619961Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"downstairs","":"downstairs","session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3892025-12-24T00:21:04.442Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.238639229Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"":"downstairs","":"downstairs","session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3902025-12-24T00:21:04.442Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.238673745Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3912025-12-24T00:21:04.442Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.238704762Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"client":"0","":"downstairs","session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3922025-12-24T00:21:04.442Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.238737888Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"client":"1","":"downstairs","session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3932025-12-24T00:21:04.442Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.238774254Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"client":"2","":"downstairs","session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3942025-12-24T00:21:04.442Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.238809811Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3952025-12-24T00:21:04.442Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.238843887Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3962025-12-24T00:21:04.442Z{"msg":"1f16c9a6-2532-4686-8173-16d9b9d6bcca is now active with session: ed26f1f2-ddd6-406c-9274-4af2c7d360e4","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.238880153Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3972025-12-24T00:21:04.442Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.238913309Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254,"session_id":"ed26f1f2-ddd6-406c-9274-4af2c7d360e4"}
3982025-12-24T00:21:04.442Z{"msg":"Disk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
3992025-12-24T00:21:04.442ZThe guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.238949155Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254}
4002025-12-24T00:21:04.442Z{"msg":"Activated sub_volume 1f16c9a6-2532-4686-8173-16d9b9d6bcca","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:03.238981512Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1254}
4012025-12-24T00:21:04.442ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4022025-12-24T00:21:04.442ZFill test
4032025-12-24T00:21:04.480ZRead and Verify all blocks (0..100 range:false)
4042025-12-24T00:21:04.557ZWrote out file "/var/tmp/test_up-build/verify_file"
4052025-12-24T00:21:04.557ZCLIENT: Tests done. All submitted work has been ACK'd
4062025-12-24T00:21:04.561Z----------------------------------------------------------------
4072025-12-24T00:21:04.561Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4082025-12-24T00:21:04.561ZStates: Active Active Active
4092025-12-24T00:21:04.561ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4102025-12-24T00:21:04.561Z 1002 Acked Read 50 Done Done Done false
4112025-12-24T00:21:04.561Z STATES DS:0 DS:1 DS:2 TOTAL
4122025-12-24T00:21:04.561Z Sent 0 0 0 0
4132025-12-24T00:21:04.561Z Done 1 1 1 3
4142025-12-24T00:21:04.561Z Skipped 0 0 0 0
4152025-12-24T00:21:04.561Z Error 0 0 0 0
4162025-12-24T00:21:04.561ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4172025-12-24T00:21:04.562ZDownstairs last five completed: 1001 1000
4182025-12-24T00:21:04.562ZUpstairs last five completed: 1002 1001 1000
4192025-12-24T00:21:04.562Z----------------------------------------------------------------
4202025-12-24T00:21:04.562Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4212025-12-24T00:21:04.562ZStates: Active Active Active
4222025-12-24T00:21:04.562ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4232025-12-24T00:21:04.562Z 1002 Acked Read 50 Sent Done Done false
4242025-12-24T00:21:04.562Z STATES DS:0 DS:1 DS:2 TOTAL
4252025-12-24T00:21:04.562Z Sent 1 0 0 1
4262025-12-24T00:21:04.562Z Done 0 1 1 2
4272025-12-24T00:21:04.562Z Skipped 0 0 0 0
4282025-12-24T00:21:04.562Z Error 0 0 0 0
4292025-12-24T00:21:04.562ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4302025-12-24T00:21:04.562ZDownstairs last five completed: 1001 1000
4312025-12-24T00:21:04.562ZUpstairs last five completed: 1002 1001 1000
4322025-12-24T00:21:04.562ZCLIENT: Up:0 ds:2 act:6
4332025-12-24T00:21:08.560Z----------------------------------------------------------------
4342025-12-24T00:21:08.560Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4352025-12-24T00:21:08.560ZDownstairs last five completed: 1003 1002 1001 1000
4362025-12-24T00:21:08.560ZUpstairs last five completed: 1003 1002 1001 1000
4372025-12-24T00:21:08.564Z----------------------------------------------------------------
4382025-12-24T00:21:08.564Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4392025-12-24T00:21:08.564ZDownstairs last five completed: 1003 1002 1001 1000
4402025-12-24T00:21:08.564ZUpstairs last five completed: 1003 1002 1001 1000
4412025-12-24T00:21:08.564ZCLIENT: Up:0 ds:0 act:6
4422025-12-24T00:21:08.564ZCLIENT: All crucible jobs finished, exiting program
4432025-12-24T00:21:08.568ZRepair part 1 passed
4442025-12-24T00:21:08.571Z
4452025-12-24T00:21:08.571ZKill the current downstairs
4462025-12-24T00:21:08.735ZDownstairs 2 stopped
4472025-12-24T00:21:08.735Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
4482025-12-24T00:21:08.738ZNow put back the original so we have a mismatch
4492025-12-24T00:21:08.738Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
4502025-12-24T00:21:08.741ZRestart downstairs with old directory
4512025-12-24T00:21:08.792Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4522025-12-24T00:21:08.811ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4532025-12-24T00:21:08.814Z 0 000-009 32 32 31 4001 4001 4000 F F F <---
4542025-12-24T00:21:08.814Z 1 010-019 32 32 31 4001 4001 4000 F F F <---
4552025-12-24T00:21:08.814Z 2 020-029 32 32 31 4001 4001 4000 F F F <---
4562025-12-24T00:21:08.814Z 3 030-039 32 32 31 4001 4001 4000 F F F <---
4572025-12-24T00:21:08.814Z 4 040-049 32 32 31 4001 4001 4000 F F F <---
4582025-12-24T00:21:08.814ZMax gen: 32, Max flush: 4001
4592025-12-24T00:21:08.814ZError: Difference in extent metadata found!
4602025-12-24T00:21:08.818Zdump test found error as expected
4612025-12-24T00:21:08.818Z
4622025-12-24T00:21:08.818Z
4632025-12-24T00:21:08.818Z/var/tmp/bins/crutest verify --range -g 33 -q --key vCBz495xm8h3swf+pJ4olsmPRhCDINMhtV6oSiYw4s8= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
4642025-12-24T00:21:08.873Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.673856398Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"task":"crutest"}
4652025-12-24T00:21:08.877Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.674533415Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"task":"crutest"}
4662025-12-24T00:21:08.877Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.676468505Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"task":"crutest"}
4672025-12-24T00:21:08.877Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.676752794Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262}
4682025-12-24T00:21:08.877Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4c4b956995d7a3110f080eebc9768bd86623f0f1\",\n git_commit_timestamp: \"2025-12-23T23:54:01.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.676801049Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262}
4692025-12-24T00:21:08.877Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.676824797Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262}
4702025-12-24T00:21:08.877Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.676847694Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262}
4712025-12-24T00:21:08.877Z{"msg":"Crucible 44b69a31-6dcd-4991-a924-749aeec4227e has session id: 94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.67743753Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
4722025-12-24T00:21:08.877Z{"msg":"Upstairs opts: Upstairs UUID: 44b69a31-6dcd-4991-a924-749aeec4227e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.677477436Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
4732025-12-24T00:21:08.877Z{"msg":"Crucible stats registered with UUID: 44b69a31-6dcd-4991-a924-749aeec4227e","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.677495734Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
4742025-12-24T00:21:08.877Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-12-24T00:21:07.67753361Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
4752025-12-24T00:21:08.877Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.67771722Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
4762025-12-24T00:21:08.877Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.677772714Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
4772025-12-24T00:21:08.877Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.677817849Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
4782025-12-24T00:21:08.881Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.679718753Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"task":"crutest"}
4792025-12-24T00:21:08.881Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.679801474Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262}
4802025-12-24T00:21:08.881Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4c4b956995d7a3110f080eebc9768bd86623f0f1\",\n git_commit_timestamp: \"2025-12-23T23:54:01.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.679848439Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262}
4812025-12-24T00:21:08.881Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.679870726Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262}
4822025-12-24T00:21:08.881Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.679900563Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262}
4832025-12-24T00:21:08.881Z{"msg":"Crucible 44b69a31-6dcd-4991-a924-749aeec4227e has session id: e52978b6-1a57-4754-b44d-5a5487ce666c","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.680239906Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
4842025-12-24T00:21:08.881Z{"msg":"Upstairs opts: Upstairs UUID: 44b69a31-6dcd-4991-a924-749aeec4227e, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.680310299Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
4852025-12-24T00:21:08.881Z{"msg":"Crucible stats registered with UUID: 44b69a31-6dcd-4991-a924-749aeec4227e","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.680367652Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
4862025-12-24T00:21:08.881Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-12-24T00:21:07.680395829Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
4872025-12-24T00:21:08.882Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.680432015Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
4882025-12-24T00:21:08.882Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.680468881Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
4892025-12-24T00:21:08.882Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.680488199Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
4902025-12-24T00:21:08.882Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.680950699Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"task":"crutest"}
4912025-12-24T00:21:08.882Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.680984855Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262}
4922025-12-24T00:21:08.882Z{"msg":"44b69a31-6dcd-4991-a924-749aeec4227e active request set","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.681064987Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
4932025-12-24T00:21:08.882Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.681097013Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"client":"0","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
4942025-12-24T00:21:08.882Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.681114851Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"client":"1","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
4952025-12-24T00:21:08.882Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.681148138Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"client":"2","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
4962025-12-24T00:21:08.882Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.681194333Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
4972025-12-24T00:21:08.882Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.6812239Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
4982025-12-24T00:21:08.882Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.681254796Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
4992025-12-24T00:21:08.882Z{"msg":"ds_connection connected from Ok(127.0.0.1:53424)","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.681286063Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5002025-12-24T00:21:08.882Z{"msg":"ds_connection connected from Ok(127.0.0.1:42960)","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.681325398Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5012025-12-24T00:21:08.882Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-12-24T00:21:07.681439496Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5022025-12-24T00:21:08.882Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-12-24T00:21:07.68149876Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5032025-12-24T00:21:08.882Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-12-24T00:21:07.681547514Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5042025-12-24T00:21:08.882Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.681574991Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5052025-12-24T00:21:08.882Z{"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.681611338Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5062025-12-24T00:21:08.886Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.682459375Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"client":"1","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5072025-12-24T00:21:08.886Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.682523948Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"client":"0","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5082025-12-24T00:21:08.886Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.682777311Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5092025-12-24T00:21:08.886Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:07.682862232Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5102025-12-24T00:21:18.884Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.682755743Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5112025-12-24T00:21:18.884Z{"msg":"ds_connection connected from Ok(127.0.0.1:59784)","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.682905717Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5122025-12-24T00:21:18.890Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.684784274Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"client":"2","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5132025-12-24T00:21:18.890Z{"msg":"[0]R flush_numbers: [4001, 4001, 4001, 4001, 4001]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.685210047Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"downstairs","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5142025-12-24T00:21:18.890Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.685267341Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"downstairs","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5152025-12-24T00:21:18.890Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.685319636Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"downstairs","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5162025-12-24T00:21:18.890Z{"msg":"[1]R flush_numbers: [4001, 4001, 4001, 4001, 4001]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.685365691Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"downstairs","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5172025-12-24T00:21:18.890Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.685409706Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"downstairs","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5182025-12-24T00:21:18.890Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.685452121Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"downstairs","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5192025-12-24T00:21:18.890Z{"msg":"[2]R flush_numbers: [4000, 4000, 4000, 4000, 4000]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.685496786Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"downstairs","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5202025-12-24T00:21:18.890Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.685539332Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"downstairs","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5212025-12-24T00:21:18.890Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.685571728Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"downstairs","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5222025-12-24T00:21:18.890Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.685613334Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"downstairs","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5232025-12-24T00:21:18.890Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.685637391Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"downstairs","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5242025-12-24T00:21:18.890Z{"msg":"Next flush: 4002","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.685666038Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"downstairs","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5252025-12-24T00:21:18.890Z{"msg":"extent 0 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4000, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.685700374Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5262025-12-24T00:21:18.890Z{"msg":"extent:0 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4000, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.6857422Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5272025-12-24T00:21:18.890Z{"msg":"found dest for source 0 for extent at index 0 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.685791724Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5282025-12-24T00:21:18.890Z{"msg":"extent 1 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4000, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.685853507Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5292025-12-24T00:21:18.890Z{"msg":"extent:1 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4000, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.685909992Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5302025-12-24T00:21:18.890Z{"msg":"found dest for source 0 for extent at index 1 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.685948127Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5312025-12-24T00:21:18.890Z{"msg":"extent 2 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4000, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.685988423Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5322025-12-24T00:21:18.890Z{"msg":"extent:2 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4000, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.686038698Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5332025-12-24T00:21:18.890Z{"msg":"found dest for source 0 for extent at index 2 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.686075994Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5342025-12-24T00:21:18.891Z{"msg":"extent 3 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4000, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.686124748Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5352025-12-24T00:21:18.891Z{"msg":"extent:3 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4000, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.686174273Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5362025-12-24T00:21:18.891Z{"msg":"found dest for source 0 for extent at index 3 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.686221618Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5372025-12-24T00:21:18.891Z{"msg":"extent 4 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4000, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.686283881Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5382025-12-24T00:21:18.891Z{"msg":"extent:4 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4001, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4000, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.686319887Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5392025-12-24T00:21:18.891Z{"msg":"found dest for source 0 for extent at index 4 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.686338535Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"mend","":"downstairs","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5402025-12-24T00:21:18.891Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.686388709Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"client":"0","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5412025-12-24T00:21:18.891Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.686441854Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"client":"1","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5422025-12-24T00:21:18.891Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.686493368Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"client":"2","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5432025-12-24T00:21:18.891Z{"msg":"Full repair list: {ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.686560311Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"downstairs","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5442025-12-24T00:21:18.894Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 4002, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 4002, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 4002, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 4002, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 4002, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.686760979Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"downstairs","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5452025-12-24T00:21:18.894Z{"msg":"starting reconciliation 4b55be84-5e2d-4cbe-98a8-97df9ab56d17: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.686783417Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"downstairs","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5462025-12-24T00:21:18.894Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.686800695Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"client":"2","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5472025-12-24T00:21:18.899Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.69864926Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"client":"2","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5482025-12-24T00:21:18.903Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.702324682Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"client":"2","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5492025-12-24T00:21:18.906Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.705944729Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"client":"2","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5502025-12-24T00:21:18.910Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.709452609Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"client":"2","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5512025-12-24T00:21:18.913Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.712694628Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"downstairs","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5522025-12-24T00:21:18.913Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.712727964Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"client":"0","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5532025-12-24T00:21:18.913Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.712747402Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"client":"1","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5542025-12-24T00:21:18.913Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.71276435Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"client":"2","":"downstairs","session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5552025-12-24T00:21:18.913Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.712781318Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5562025-12-24T00:21:18.913Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.712807396Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5572025-12-24T00:21:18.914Z{"msg":"44b69a31-6dcd-4991-a924-749aeec4227e is now active with session: 94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.712851761Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"session_id":"94eaf0b4-fbb9-47c4-b0e8-377c91c6ed9c"}
5582025-12-24T00:21:18.914Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.712890727Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262}
5592025-12-24T00:21:18.914Z{"msg":"Activated sub_volume 44b69a31-6dcd-4991-a924-749aeec4227e","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.712916864Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262}
5602025-12-24T00:21:18.918Z{"msg":"44b69a31-6dcd-4991-a924-749aeec4227e active request set","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.712986616Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5612025-12-24T00:21:18.918Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.713078656Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262}
5622025-12-24T00:21:18.918Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.713143819Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"client":"0","":"downstairs","session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5632025-12-24T00:21:18.918Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.713181525Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"client":"1","":"downstairs","session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5642025-12-24T00:21:18.918Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.713217741Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"client":"2","":"downstairs","session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5652025-12-24T00:21:18.918Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.713249708Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5662025-12-24T00:21:18.918Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.713281694Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5672025-12-24T00:21:18.918Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.713313011Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5682025-12-24T00:21:18.918Z{"msg":"ds_connection connected from Ok(127.0.0.1:49593)","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.713345387Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5692025-12-24T00:21:18.918Z{"msg":"ds_connection connected from Ok(127.0.0.1:41998)","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.713379403Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5702025-12-24T00:21:18.918Z{"msg":"ds_connection connected from Ok(127.0.0.1:36172)","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.71340958Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5712025-12-24T00:21:18.918Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.71387041Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"client":"0","":"downstairs","session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5722025-12-24T00:21:18.918Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.713948652Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"client":"1","":"downstairs","session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5732025-12-24T00:21:18.918Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.714006326Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"client":"2","":"downstairs","session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5742025-12-24T00:21:18.918Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.714201514Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5752025-12-24T00:21:18.918Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.714301283Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5762025-12-24T00:21:18.918Z{"msg":"[0]R flush_numbers: [5322, 5322, 5322, 5322, 5322]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.714344769Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"downstairs","":"downstairs","session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5772025-12-24T00:21:18.918Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.714385714Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"downstairs","":"downstairs","session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5782025-12-24T00:21:18.918Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.71442462Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"downstairs","":"downstairs","session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5792025-12-24T00:21:18.918Z{"msg":"[1]R flush_numbers: [5322, 5322, 5322, 5322, 5322]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.714462866Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"downstairs","":"downstairs","session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5802025-12-24T00:21:18.918Z{"msg":"Wait for a query_work_queue command to finish before sending IO
5812025-12-24T00:21:18.918Z[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.714507691Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"downstairs","":"downstairs","session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5822025-12-24T00:21:18.918Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.714546387Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"downstairs","":"downstairs","session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5832025-12-24T00:21:18.918Z{"msg":"[2]R flush_numbers: [5322, 5322, 5322, 5322, 5322]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.714587772Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"downstairs","":"downstairs","session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5842025-12-24T00:21:18.918Z{"msg":"[2]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.714620109Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"downstairs","":"downstairs","session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5852025-12-24T00:21:18.918Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.714658005Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"downstairs","":"downstairs","session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5862025-12-24T00:21:18.918Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.714678253Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"downstairs","":"downstairs","session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5872025-12-24T00:21:18.918Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.714716069Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"downstairs","":"downstairs","session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5882025-12-24T00:21:18.918Z{"msg":"Next flush: 5323","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.714757294Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"downstairs","":"downstairs","session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5892025-12-24T00:21:18.918Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.71479468Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"":"downstairs","":"downstairs","session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5902025-12-24T00:21:18.918Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.714831156Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5912025-12-24T00:21:18.918Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.714863413Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"client":"0","":"downstairs","session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5922025-12-24T00:21:18.919Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.714898249Z","hostname":"Disk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
5932025-12-24T00:21:18.919Zw-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"client":"1","":"downstairs","session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5942025-12-24T00:21:18.919Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.714933385Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"client":"2","":"downstairs","session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5952025-12-24T00:21:18.919Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.714969181Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5962025-12-24T00:21:18.919Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.715003238Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5972025-12-24T00:21:18.919Z{"msg":"44b69a31-6dcd-4991-a924-749aeec4227e is now active with session: e52978b6-1a57-4754-b44d-5a5487ce666c","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.715023705Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5982025-12-24T00:21:18.919Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.715040034Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262,"session_id":"e52978b6-1a57-4754-b44d-5a5487ce666c"}
5992025-12-24T00:21:18.919Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.71507308Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262}
6002025-12-24T00:21:18.919ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
6012025-12-24T00:21:18.919Z{"msg":"Activated sub_volume 44b69a31-6dcd-4991-a924-749aeec4227e","v":0,"name":"crucible","level":30,"time":"2025-12-24T00:21:17.715099147Z","hostname":"w-01KD6V8WYBDEH5N3JQ2EV305S3","pid":1262}
6022025-12-24T00:21:18.919ZRead and Verify all blocks (0..100 range:true)
6032025-12-24T00:21:19.003ZWrote out file "/var/tmp/test_up-build/verify_file"
6042025-12-24T00:21:19.003ZVerify test completed
6052025-12-24T00:21:19.007ZWrote out file "/var/tmp/test_up-build/verify_file"
6062025-12-24T00:21:19.007ZCLIENT: Tests done. All submitted work has been ACK'd
6072025-12-24T00:21:19.007Z----------------------------------------------------------------
6082025-12-24T00:21:19.007Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6092025-12-24T00:21:19.007ZStates: Active Active Active
6102025-12-24T00:21:19.007ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6112025-12-24T00:21:19.007Z 1000 Acked Read 50 Done Done Done false
6122025-12-24T00:21:19.007Z STATES DS:0 DS:1 DS:2 TOTAL
6132025-12-24T00:21:19.007Z Sent 0 0 0 0
6142025-12-24T00:21:19.007Z Done 1 1 1 3
6152025-12-24T00:21:19.007Z Skipped 0 0 0 0
6162025-12-24T00:21:19.007Z Error 0 0 0 0
6172025-12-24T00:21:19.007ZLast Flush: None None None
6182025-12-24T00:21:19.007ZDownstairs last five completed:
6192025-12-24T00:21:19.007ZUpstairs last five completed: 1000
6202025-12-24T00:21:19.007Z----------------------------------------------------------------
6212025-12-24T00:21:19.007Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6222025-12-24T00:21:19.007ZStates: Active Active Active
6232025-12-24T00:21:19.007ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6242025-12-24T00:21:19.007Z 1000 Acked Read 50 Done Done Done false
6252025-12-24T00:21:19.007Z STATES DS:0 DS:1 DS:2 TOTAL
6262025-12-24T00:21:19.007Z Sent 0 0 0 0
6272025-12-24T00:21:19.007Z Done 1 1 1 3
6282025-12-24T00:21:19.007Z Skipped 0 0 0 0
6292025-12-24T00:21:19.007Z Error 0 0 0 0
6302025-12-24T00:21:19.007ZLast Flush: None None None
6312025-12-24T00:21:19.007ZDownstairs last five completed:
6322025-12-24T00:21:19.007ZUpstairs last five completed: 1000
6332025-12-24T00:21:19.007ZCLIENT: Up:0 ds:2 act:6
6342025-12-24T00:21:23.004Z----------------------------------------------------------------
6352025-12-24T00:21:23.004Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6362025-12-24T00:21:23.004ZDownstairs last five completed: 1001 1000
6372025-12-24T00:21:23.004ZUpstairs last five completed: 1001 1000
6382025-12-24T00:21:23.009Z----------------------------------------------------------------
6392025-12-24T00:21:23.009Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6402025-12-24T00:21:23.009ZDownstairs last five completed: 1001 1000
6412025-12-24T00:21:23.009ZUpstairs last five completed: 1001 1000
6422025-12-24T00:21:23.009ZCLIENT: Up:0 ds:0 act:6
6432025-12-24T00:21:23.009ZCLIENT: All crucible jobs finished, exiting program
6442025-12-24T00:21:23.012ZRepair part 2 passed
6452025-12-24T00:21:23.012Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
6462025-12-24T00:21:23.030ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6472025-12-24T00:21:23.030Z 0 000-009 32 32 32 4001 4001 4001 F F F
6482025-12-24T00:21:23.033Z 1 010-019 32 32 32 4001 4001 4001 F F F
6492025-12-24T00:21:23.033Z 2 020-029 32 32 32 4001 4001 4001 F F F
6502025-12-24T00:21:23.033Z 3 030-039 32 32 32 4001 4001 4001 F F F
6512025-12-24T00:21:23.034Z 4 040-049 32 32 32 4001 4001 4001 F F F
6522025-12-24T00:21:23.034ZMax gen: 32, Max flush: 4001
6532025-12-24T00:21:23.037Zdump test passed
6542025-12-24T00:21:23.037Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
6552025-12-24T00:21:23.050Z Extent 1
6562025-12-24T00:21:23.054ZGEN 32 32 32
6572025-12-24T00:21:23.054ZFLUSH_ID 4001 4001 4001
6582025-12-24T00:21:23.054ZDIRTY
6592025-12-24T00:21:23.054Z
6602025-12-24T00:21:23.054ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
6612025-12-24T00:21:23.054Z010 A A A A A A
6622025-12-24T00:21:23.054Z011 A A A A A A
6632025-12-24T00:21:23.057Z012 A A A A A A
6642025-12-24T00:21:23.057Z013 A A A A A A
6652025-12-24T00:21:23.061Z014 A A A A A A
6662025-12-24T00:21:23.061Z015 A A A A A A
6672025-12-24T00:21:23.061Z016 A A A A A A
6682025-12-24T00:21:23.064Z017 A A A A A A
6692025-12-24T00:21:23.064Z018 A A A A A A
6702025-12-24T00:21:23.067Z019 A A A A A A
6712025-12-24T00:21:23.071Zdump extent test passed
6722025-12-24T00:21:23.071Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
6732025-12-24T00:21:23.085ZExtent 2 Block in extent 0 Actual block 20
6742025-12-24T00:21:23.089Z
6752025-12-24T00:21:23.090Z DATA SHA256 VER
6762025-12-24T00:21:23.090Z------ ---------------------------------------------------------------- ---
6772025-12-24T00:21:23.090Z 0 1647a4278e97efa5e93ba434c3dae1f52af3d9de31e882daf4363d86c74c48ee A
6782025-12-24T00:21:23.090Z 1 1647a4278e97efa5e93ba434c3dae1f52af3d9de31e882daf4363d86c74c48ee A
6792025-12-24T00:21:23.090Z 2 1647a4278e97efa5e93ba434c3dae1f52af3d9de31e882daf4363d86c74c48ee A
6802025-12-24T00:21:23.090Z
6812025-12-24T00:21:23.090ZNONCES 0 1 2 DIFF
6822025-12-24T00:21:23.090Z------ ------------------------ ------------------------ ------------------------ -----
6832025-12-24T00:21:23.090Z 0 2d327003532082efcf4be4dd 2d327003532082efcf4be4dd 2d327003532082efcf4be4dd
6842025-12-24T00:21:23.090Z
6852025-12-24T00:21:23.090Z TAGS 0 1 2 DIFF
6862025-12-24T00:21:23.090Z------ -------------------------------- -------------------------------- -------------------------------- -----
6872025-12-24T00:21:23.090Z 0 c9cfb36f4b98b90f18db1d75daf67ea0 c9cfb36f4b98b90f18db1d75daf67ea0 c9cfb36f4b98b90f18db1d75daf67ea0
6882025-12-24T00:21:23.090Z
6892025-12-24T00:21:23.090ZHASHES 0 1 2 DIFF
6902025-12-24T00:21:23.090Z------ ---------------- ---------------- ---------------- -----
6912025-12-24T00:21:23.090Z
6922025-12-24T00:21:23.094Zdump block test passed
6932025-12-24T00:21:23.094ZInitial upstairs tests have completed, stopping all downstairs
6942025-12-24T00:21:28.158ZCreating 7 larger downstairs regions
6952025-12-24T00:21:28.508ZStarting 7 downstairs
6962025-12-24T00:21:33.517Zdsc restarted at PID: 1278
6972025-12-24T00:21:33.904ZNow do the replace-reconcile test
6982025-12-24T00:21:33.956ZUsing 8870 for the replacement port
6992025-12-24T00:25:32.228ZCompleted test: replace-reconcile
7002025-12-24T00:25:32.231ZNow do the replace-before-active test
7012025-12-24T00:31:38.420ZCompleted test: replace-before-active
7022025-12-24T00:31:38.424ZAll tests have completed, stopping all downstairs
7032025-12-24T00:31:38.481Z
7042025-12-24T00:31:38.481ZAll Tests have passed
7052025-12-24T00:31:38.484Z16:22 Test duration
7062025-12-24T00:31:38.484Z
7072025-12-24T00:31:38.484Zreal 16:21.501214657
7082025-12-24T00:31:38.484Zuser 40:49.992186615
7092025-12-24T00:31:38.484Zsys 30.184447274
7102025-12-24T00:31:38.484Ztrap 1.306196772
7112025-12-24T00:31:38.484Ztflt 0.050307773
7122025-12-24T00:31:38.484Zdflt 0.093464240
7132025-12-24T00:31:38.484Zkflt 0.001285490
7142025-12-24T00:31:38.484Zlock 23:45:36.460740491
7152025-12-24T00:31:38.484Zslp 1:13:35.109124206
7162025-12-24T00:31:38.484Zlat 5:16.021900327
7172025-12-24T00:31:38.484Zstop 0.081865269
7182025-12-24T00:31:38.484Z+ echo 'test-up-2region-encrypted ends'
7192025-12-24T00:31:38.485Ztest-up-2region-encrypted ends
7202025-12-24T00:31:43.488Zprocess exited: duration 991739 ms, exit code 0
7212025-12-24T00:31:43.488Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
7222025-12-24T00:32:43.499Zexec warning: : stdout descriptor may be held open by a background process; giving up!
7232025-12-24T00:32:43.499Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
7242025-12-24T00:32:43.513Zfound 17 output files
7252025-12-24T00:32:43.513Zuploading: /tmp/test_up-build/dsc-out.txt (57797 bytes)
7262025-12-24T00:32:44.530Zuploaded: /tmp/test_up-build/dsc-out.txt
7272025-12-24T00:32:44.530Zuploading: /tmp/test_up-build/test_up_out.txt (22574601 bytes)
7282025-12-24T00:32:45.747Zuploaded: /tmp/test_up-build/test_up_out.txt
7292025-12-24T00:32:45.752Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (550921 bytes)
7302025-12-24T00:32:46.773Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
7312025-12-24T00:32:46.777Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (548309 bytes)
7322025-12-24T00:32:47.800Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
7332025-12-24T00:32:47.803Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (979505 bytes)
7342025-12-24T00:32:48.831Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
7352025-12-24T00:32:48.835Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (543038 bytes)
7362025-12-24T00:32:49.856Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
7372025-12-24T00:32:49.859Zuploading: /tmp/test_up-build/dsc/downstairs-8850.txt (2480 bytes)
7382025-12-24T00:32:50.867Zuploaded: /tmp/test_up-build/dsc/downstairs-8850.txt
7392025-12-24T00:32:50.867Zuploading: /tmp/test_up-build/dsc/downstairs-8860.txt (44671 bytes)
7402025-12-24T00:32:51.882Zuploaded: /tmp/test_up-build/dsc/downstairs-8860.txt
7412025-12-24T00:32:51.885Zuploading: /tmp/test_up-build/dsc/downstairs-8870.txt (3410096 bytes)
7422025-12-24T00:32:52.931Zuploaded: /tmp/test_up-build/dsc/downstairs-8870.txt
7432025-12-24T00:32:52.934Zuploading: /tmp/debug/df.txt (1270 bytes)
7442025-12-24T00:32:53.945Zuploaded: /tmp/debug/df.txt
7452025-12-24T00:32:53.946Zuploading: /tmp/debug/dtrace.txt (1405391 bytes)
7462025-12-24T00:32:54.980Zuploaded: /tmp/debug/dtrace.txt
7472025-12-24T00:32:54.983Zuploading: /tmp/debug/iostat.txt (315402 bytes)
7482025-12-24T00:32:54.999Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 315402 -> 319014
7492025-12-24T00:32:56.007Zuploaded: /tmp/debug/iostat.txt
7502025-12-24T00:32:56.007Zuploading: /tmp/debug/mpstat.txt (764343 bytes)
7512025-12-24T00:32:56.029Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 764343 -> 773833
7522025-12-24T00:32:57.037Zuploaded: /tmp/debug/mpstat.txt
7532025-12-24T00:32:57.040Zuploading: /tmp/debug/paging.txt (138042 bytes)
7542025-12-24T00:32:57.052Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 138042 -> 139982
7552025-12-24T00:32:58.063Zuploaded: /tmp/debug/paging.txt
7562025-12-24T00:32:58.067Zuploading: /tmp/debug/prstat.txt (1512570 bytes)
7572025-12-24T00:32:58.092Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 1512570 -> 1530676
7582025-12-24T00:32:59.100Zuploaded: /tmp/debug/prstat.txt
7592025-12-24T00:32:59.103Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
7602025-12-24T00:33:00.115Zuploaded: /tmp/debug/psrinfo.txt
7612025-12-24T00:33:00.115Zuploading: /tmp/debug/upstairs-info.txt (236572 bytes)
7622025-12-24T00:33:01.139Zuploaded: /tmp/debug/upstairs-info.txt