01K7M84X8TD07FWQ1KQT4P2YHC: test-up-2region-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01K7M85G5VNBQWVE6DAMJ5RXYA

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12025-10-15T15:55:18.460Zjob dependencies complete; ready to run (waiting for 20 m 11 s)
22025-10-15T15:56:17.182Zjob assigned to worker 01K7M9AKW94XMDZXJKNDC1H832 [factory aws, i-0529766283150de7e] (queued for 58 s)
32025-10-15T15:56:25.104Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42025-10-15T15:56:27.319Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52025-10-15T15:56:27.319Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62025-10-15T15:56:29.386Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72025-10-15T15:56:29.386Zdownloading input: /input/build/work/bins/crutest.gz
82025-10-15T15:56:30.960Zdownloaded input: /input/build/work/bins/crutest.gz
92025-10-15T15:56:30.960Zdownloading input: /input/build/work/bins/dsc.gz
102025-10-15T15:56:31.653Zdownloaded input: /input/build/work/bins/dsc.gz
112025-10-15T15:56:31.657Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122025-10-15T15:56:31.676Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132025-10-15T15:56:31.676Zdownloading input: /input/build/work/scripts/test_ds.sh
142025-10-15T15:56:31.702Zdownloaded input: /input/build/work/scripts/test_ds.sh
152025-10-15T15:56:31.702Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162025-10-15T15:56:31.725Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172025-10-15T15:56:31.725Zdownloading input: /input/build/work/scripts/test_repair.sh
182025-10-15T15:56:31.748Zdownloaded input: /input/build/work/scripts/test_repair.sh
192025-10-15T15:56:31.748Zdownloading input: /input/build/work/scripts/test_replay.sh
202025-10-15T15:56:31.771Zdownloaded input: /input/build/work/scripts/test_replay.sh
212025-10-15T15:56:31.772Zdownloading input: /input/build/work/scripts/test_up.sh
222025-10-15T15:56:31.793Zdownloaded input: /input/build/work/scripts/test_up.sh
232025-10-15T15:56:31.793Zdownloading input: /input/build/work/scripts/upstairs_info.d
242025-10-15T15:56:31.817Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252025-10-15T15:56:31.822Zdownloading input: /input/build/tmp/cargo-test-out.log
262025-10-15T15:56:32.358Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272025-10-15T15:56:32.358Zstarting task 0: "setup"
282025-10-15T15:56:32.366Z++ uname -s
292025-10-15T15:56:32.370Z+ kern=SunOS
302025-10-15T15:56:32.370Z+ build_user=build
312025-10-15T15:56:32.370Z+ build_uid=12345
322025-10-15T15:56:32.370Z+ work_dir=/work
332025-10-15T15:56:32.370Z+ input_dir=/input
342025-10-15T15:56:32.370Z+ [[ 0 == 12345 ]]
352025-10-15T15:56:32.370Z+ case "$kern" in
362025-10-15T15:56:32.370Z+ groupadd -g 12345 build
372025-10-15T15:56:32.374Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382025-10-15T15:56:34.378Z+ zfs create -o mountpoint=/work rpool/work
392025-10-15T15:56:34.769Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402025-10-15T15:56:34.773Z+ home_fs=zfs
412025-10-15T15:56:34.773Z+ [[ zfs == autofs ]]
422025-10-15T15:56:34.773Z+ mkdir -p /home/build
432025-10-15T15:56:34.776Z+ chown build:build /home/build /work
442025-10-15T15:56:35.776Z+ chmod 0700 /home/build /work
452025-10-15T15:56:35.779Zprocess exited: duration 3419 ms, exit code 0
 
462025-10-15T15:56:35.786Zstarting task 1: "authentication"
472025-10-15T15:56:35.807Zprocess exited: duration 20 ms, exit code 0
 
482025-10-15T15:56:35.818Zstarting task 2: "build"
492025-10-15T15:56:35.822Z+ banner cores
502025-10-15T15:56:35.825Z
512025-10-15T15:56:35.826Z #### #### ##### ###### ####
522025-10-15T15:56:35.826Z # # # # # # # #
532025-10-15T15:56:35.826Z # # # # # ##### ####
542025-10-15T15:56:35.826Z # # # ##### # #
552025-10-15T15:56:35.826Z # # # # # # # # #
562025-10-15T15:56:35.826Z #### #### # # ###### ####
572025-10-15T15:56:35.826Z
582025-10-15T15:56:35.826Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
592025-10-15T15:56:35.834Z+ banner unpack
602025-10-15T15:56:35.838Z
612025-10-15T15:56:35.838Z # # # # ##### ## #### # #
622025-10-15T15:56:35.838Z # # ## # # # # # # # # #
632025-10-15T15:56:35.838Z # # # # # # # # # # ####
642025-10-15T15:56:35.838Z # # # # # ##### ###### # # #
652025-10-15T15:56:35.838Z # # # ## # # # # # # #
662025-10-15T15:56:35.838Z #### # # # # # #### # #
672025-10-15T15:56:35.838Z
682025-10-15T15:56:35.838Z+ mkdir -p /var/tmp/bins
692025-10-15T15:56:35.842Z+ for t in "$input/bins/"*.gz
702025-10-15T15:56:35.842Z++ basename /input/build/work/bins/crucible-downstairs.gz
712025-10-15T15:56:35.842Z+ b=crucible-downstairs.gz
722025-10-15T15:56:35.842Z+ b=crucible-downstairs
732025-10-15T15:56:35.842Z+ gunzip
742025-10-15T15:56:38.740Z+ chmod +x /var/tmp/bins/crucible-downstairs
752025-10-15T15:56:38.744Z+ for t in "$input/bins/"*.gz
762025-10-15T15:56:38.744Z++ basename /input/build/work/bins/crucible-hammer.gz
772025-10-15T15:56:38.744Z+ b=crucible-hammer.gz
782025-10-15T15:56:38.748Z+ b=crucible-hammer
792025-10-15T15:56:38.748Z+ gunzip
802025-10-15T15:56:41.516Z+ chmod +x /var/tmp/bins/crucible-hammer
812025-10-15T15:56:41.527Z+ for t in "$input/bins/"*.gz
822025-10-15T15:56:41.528Z++ basename /input/build/work/bins/crutest.gz
832025-10-15T15:56:41.528Z+ b=crutest.gz
842025-10-15T15:56:41.528Z+ b=crutest
852025-10-15T15:56:41.528Z+ gunzip
862025-10-15T15:56:44.427Z+ chmod +x /var/tmp/bins/crutest
872025-10-15T15:56:44.437Z+ for t in "$input/bins/"*.gz
882025-10-15T15:56:44.437Z++ basename /input/build/work/bins/dsc.gz
892025-10-15T15:56:44.437Z+ b=dsc.gz
902025-10-15T15:56:44.437Z+ b=dsc
912025-10-15T15:56:44.437Z+ gunzip
922025-10-15T15:56:45.801Z+ chmod +x /var/tmp/bins/dsc
932025-10-15T15:56:45.806Z+ export BINDIR=/var/tmp/bins
942025-10-15T15:56:45.806Z+ BINDIR=/var/tmp/bins
952025-10-15T15:56:45.806Z+ jobpid=1155
962025-10-15T15:56:45.806Z+ echo 'Setup debug logging'
972025-10-15T15:56:45.806ZSetup debug logging
982025-10-15T15:56:45.806Z+ mkdir /tmp/debug
992025-10-15T15:56:45.806Z+ sleep 3600
1002025-10-15T15:56:45.810Z+ psrinfo -v
1012025-10-15T15:56:45.810Z+ df -h
1022025-10-15T15:56:45.818Z+ prstat -d d -mLc 1
1032025-10-15T15:56:45.822Z+ iostat -T d -xn 1
1042025-10-15T15:56:45.822Z+ mpstat -T d 1
1052025-10-15T15:56:45.822Z+ vmstat -T d -p 1
1062025-10-15T15:56:45.822Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1072025-10-15T15:56:45.822Z+ banner test_up_2r_encrypted
1082025-10-15T15:56:45.822Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1092025-10-15T15:56:45.826Z #####
1102025-10-15T15:56:45.826Z ##### ###### #### ##### # # ##### # # #####
1112025-10-15T15:56:45.826Z # # # # # # # # # # #
1122025-10-15T15:56:45.826Z # ##### #### # # # # # ##### # #
1132025-10-15T15:56:45.826Z # # # # # # ##### # #####
1142025-10-15T15:56:45.826Z # # # # # # # # # # #
1152025-10-15T15:56:45.826Z # ###### #### # ####### #### # ####### ####### # #
1162025-10-15T15:56:45.827Z
1172025-10-15T15:56:45.827Z+ ptime -m bash /input/build/work/scripts/test_up.sh -r 2 -N encrypted
1182025-10-15T15:56:45.836Z/input/build/work
1192025-10-15T15:56:45.848ZUsing 2 region sets
1202025-10-15T15:56:45.848ZTurn off color for downstairs dump
1212025-10-15T15:56:45.864ZUpstairs using key: bjJf7udwLUTuGZvKPtAEtzbHDlsPbeDGqK7OCMXKwN4=
1222025-10-15T15:56:45.868Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1232025-10-15T15:56:45.868ZCreating 6 downstairs regions
1242025-10-15T15:56:46.250ZStarting 6 downstairs
1252025-10-15T15:56:46.254Zdsc started at PID: 1203
1262025-10-15T15:56:51.607ZDisable automatic restart on all downstairs
1272025-10-15T15:56:51.663Z
1282025-10-15T15:56:51.663ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1292025-10-15T15:56:51.666ZRunning test: span
1302025-10-15T15:56:55.858ZCompleted test: span
1312025-10-15T15:57:00.867ZRunning test: big
1322025-10-15T15:57:05.452ZCompleted test: big
1332025-10-15T15:57:10.460ZRunning test: dep
1342025-10-15T15:57:44.182ZCompleted test: dep
1352025-10-15T15:57:49.189ZRunning test: balloon
1362025-10-15T16:02:04.235ZCompleted test: balloon
1372025-10-15T16:02:09.242ZRunning test: deactivate
1382025-10-15T16:02:13.888ZCompleted test: deactivate
1392025-10-15T16:02:18.895ZRunning hammer
1402025-10-15T16:02:32.029ZRun repair tests
1412025-10-15T16:02:32.033Z/var/tmp/bins/crutest fill -g 31 -q --key bjJf7udwLUTuGZvKPtAEtzbHDlsPbeDGqK7OCMXKwN4= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1422025-10-15T16:02:32.093Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.642775406Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"task":"crutest"}
1432025-10-15T16:02:32.097Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.643419259Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"task":"crutest"}
1442025-10-15T16:02:32.097Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.645227767Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"task":"crutest"}
1452025-10-15T16:02:32.097Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.645442275Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246}
1462025-10-15T16:02:32.097Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"13b1577b584ca0dbd09b897d333d5c2b128ce008\",\n git_commit_timestamp: \"2025-10-15T15:33:57.000000000Z\",\n git_branch: \"mkeeter/check-hashes\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.645503302Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246}
1472025-10-15T16:02:32.097Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.64553318Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246}
1482025-10-15T16:02:32.097Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.645561078Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246}
1492025-10-15T16:02:32.097Z{"msg":"Crucible 0cf2aaa9-507e-4bfa-8d92-45152e3c2633 has session id: 7394a66e-d0c8-441d-9b6b-8bb929ff7e71","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.646150945Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1502025-10-15T16:02:32.097Z{"msg":"Upstairs opts: Upstairs UUID: 0cf2aaa9-507e-4bfa-8d92-45152e3c2633, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.646224511Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1512025-10-15T16:02:32.097Z{"msg":"Crucible stats registered with UUID: 0cf2aaa9-507e-4bfa-8d92-45152e3c2633","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.646262569Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1522025-10-15T16:02:32.097Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-10-15T16:02:31.646285488Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1532025-10-15T16:02:32.097Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.646530104Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1542025-10-15T16:02:32.097Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.646583251Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1552025-10-15T16:02:32.097Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.646614069Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1562025-10-15T16:02:32.101Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.64853202Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"task":"crutest"}
1572025-10-15T16:02:32.101Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.648600597Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246}
1582025-10-15T16:02:32.101Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"13b1577b584ca0dbd09b897d333d5c2b128ce008\",\n git_commit_timestamp: \"2025-10-15T15:33:57.000000000Z\",\n git_branch: \"mkeeter/check-hashes\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.648663963Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246}
1592025-10-15T16:02:32.101Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.648688512Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246}
1602025-10-15T16:02:32.101Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.64871083Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246}
1612025-10-15T16:02:32.101Z{"msg":"Crucible 0cf2aaa9-507e-4bfa-8d92-45152e3c2633 has session id: e82eead9-98bf-4a56-bc3f-bc52b3159e68","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.649013303Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
1622025-10-15T16:02:32.101Z{"msg":"Upstairs opts: Upstairs UUID: 0cf2aaa9-507e-4bfa-8d92-45152e3c2633, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.64907469Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
1632025-10-15T16:02:32.101Z{"msg":"Crucible stats registered with UUID: 0cf2aaa9-507e-4bfa-8d92-45152e3c2633","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.649104118Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
1642025-10-15T16:02:32.101Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-10-15T16:02:31.649147906Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
1652025-10-15T16:02:32.101Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.649183304Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
1662025-10-15T16:02:32.101Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.649209162Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
1672025-10-15T16:02:32.101Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.649227361Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
1682025-10-15T16:02:32.101Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.649965929Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"task":"crutest"}
1692025-10-15T16:02:32.101Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.649993008Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246}
1702025-10-15T16:02:32.101Z{"msg":"0cf2aaa9-507e-4bfa-8d92-45152e3c2633 active request set","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.650056054Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1712025-10-15T16:02:32.101Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.650105481Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"client":"0","":"downstairs","session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1722025-10-15T16:02:32.101Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.65013337Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"client":"1","":"downstairs","session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1732025-10-15T16:02:32.101Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.650176057Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"client":"2","":"downstairs","session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1742025-10-15T16:02:32.101Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.650198126Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1752025-10-15T16:02:32.101Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.650229394Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1762025-10-15T16:02:32.101Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.650261923Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1772025-10-15T16:02:32.102Z{"msg":"ds_connection connected from Ok(127.0.0.1:37690)","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.650306Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1782025-10-15T16:02:32.102Z{"msg":"ds_connection connected from Ok(127.0.0.1:42328)","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.650345668Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1792025-10-15T16:02:32.102Z{"msg":"ds_connection connected from Ok(127.0.0.1:39210)","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.650382806Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1802025-10-15T16:02:32.107Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.651730949Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"client":"2","":"downstairs","session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1812025-10-15T16:02:32.107Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.651861952Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"client":"1","":"downstairs","session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1822025-10-15T16:02:32.107Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.65190408Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"client":"0","":"downstairs","session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1832025-10-15T16:02:32.107Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.652146606Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1842025-10-15T16:02:32.107Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.652231741Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1852025-10-15T16:02:32.107Z{"msg":"[0]R flush_numbers: [4001, 4001, 4001, 4001, 4001]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.652326356Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"downstairs","":"downstairs","session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1862025-10-15T16:02:32.107Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.652357994Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"downstairs","":"downstairs","session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1872025-10-15T16:02:32.107Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.652376753Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"downstairs","":"downstairs","session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1882025-10-15T16:02:32.107Z{"msg":"[1]R flush_numbers: [4001, 4001, 4001, 4001, 4001]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.65243255Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"downstairs","":"downstairs","session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1892025-10-15T16:02:32.107Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.652451199Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"downstairs","":"downstairs","session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1902025-10-15T16:02:32.107Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.652475327Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"downstairs","":"downstairs","session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1912025-10-15T16:02:32.107Z{"msg":"[2]R flush_numbers: [4001, 4001, 4001, 4001, 4001]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.652492717Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"downstairs","":"downstairs","session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1922025-10-15T16:02:32.107Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.652509056Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"downstairs","":"downstairs","session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1932025-10-15T16:02:32.107Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.652539044Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"downstairs","":"downstairs","session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1942025-10-15T16:02:32.107Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.652566342Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"downstairs","":"downstairs","session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1952025-10-15T16:02:32.107Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.6525988Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"downstairs","":"downstairs","session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1962025-10-15T16:02:32.107Z{"msg":"Next flush: 4002","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.652628009Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"downstairs","":"downstairs","session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1972025-10-15T16:02:32.107Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.652667116Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"downstairs","":"downstairs","session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1982025-10-15T16:02:32.107Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.652712054Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
1992025-10-15T16:02:32.107Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.652745662Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"client":"0","":"downstairs","session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
2002025-10-15T16:02:32.107Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.65277966Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"client":"1","":"downstairs","session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
2012025-10-15T16:02:32.107Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.652811948Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"client":"2","":"downstairs","session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
2022025-10-15T16:02:32.107Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.652842847Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
2032025-10-15T16:02:32.107Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.6529636Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
2042025-10-15T16:02:32.107Z{"msg":"0cf2aaa9-507e-4bfa-8d92-45152e3c2633 is now active with session: 7394a66e-d0c8-441d-9b6b-8bb929ff7e71","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.653008187Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
2052025-10-15T16:02:32.107Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.653026356Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"session_id":"7394a66e-d0c8-441d-9b6b-8bb929ff7e71"}
2062025-10-15T16:02:32.107Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.653065194Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246}
2072025-10-15T16:02:32.107Z{"msg":"Activated sub_volume 0cf2aaa9-507e-4bfa-8d92-45152e3c2633","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.653091723Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246}
2082025-10-15T16:02:32.107Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.653143969Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246}
2092025-10-15T16:02:32.107Z{"msg":"0cf2aaa9-507e-4bfa-8d92-45152e3c2633 active request set","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.653185547Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2102025-10-15T16:02:32.107Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.653218615Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"client":"0","":"downstairs","session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2112025-10-15T16:02:32.107Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.653251894Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"client":"1","":"downstairs","session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2122025-10-15T16:02:32.107Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.653295141Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"client":"2","":"downstairs","session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2132025-10-15T16:02:32.107Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.653342098Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2142025-10-15T16:02:32.108Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.653377676Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2152025-10-15T16:02:32.108Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.653423444Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2162025-10-15T16:02:32.108Z{"msg":"ds_connection connected from Ok(127.0.0.1:52204)","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.653462462Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2172025-10-15T16:02:32.108Z{"msg":"ds_connection connected from Ok(127.0.0.1:45119)","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.65349637Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2182025-10-15T16:02:32.108Z{"msg":"ds_connection connected from Ok(127.0.0.1:57516)","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.653536557Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2192025-10-15T16:02:32.108Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.653868169Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"client":"1","":"downstairs","session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2202025-10-15T16:02:32.108Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.65401218Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"client":"0","":"downstairs","session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2212025-10-15T16:02:32.108Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.654046858Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"client":"2","":"downstairs","session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2222025-10-15T16:02:32.108Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.654249637Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2232025-10-15T16:02:32.108Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.654388489Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2242025-10-15T16:02:32.108Z{"msg":"[0]R flush_numbers: [5322, 5324, 5322, 5325, 5325]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.654469115Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"downstairs","":"downstairs","session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2252025-10-15T16:02:32.108Z{"msg":"[0]R generation: [13, 14, 13, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.654500053Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"downstairs","":"downstairs","session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2262025-10-15T16:02:32.108Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.654530561Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"downstairs","":"downstairs","session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2272025-10-15T16:02:32.108Z{"msg":"[1]R flush_numbers: [5322, 5324, 5322, 5325, 5325]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.65455128Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"downstairs","":"downstairs","session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2282025-10-15T16:02:32.108Z{"msg":"[1]R generation: [13, 14, 13, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.654580068Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"downstairs","":"downstairs","session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2292025-10-15T16:02:32.108Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.654597977Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"downstairs","":"downstairs","session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2302025-10-15T16:02:32.108Z{"msg":"[2]R flush_numbers: [5322, 5324, 5322, 5325, 5325]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.654621616Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"downstairs","":"downstairs","session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2312025-10-15T16:02:32.108Z{"msg":"[2]R generation: [13, 14, 13, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.654639095Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"downstairs","":"downstairs","session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2322025-10-15T16:02:32.108Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.654662364Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"downstairs","":"downstairs","session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2332025-10-15T16:02:32.108Z{"msg":"Max found gen is 15","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.654678493Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"downstairs","":"downstairs","session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2342025-10-15T16:02:32.108Z{"msg":"Generation requested: 31 >= found:15","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.654695742Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"downstairs","":"downstairs","session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2352025-10-15T16:02:32.108Z{"msg":"Next flush: 5326","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.654712881Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"downstairs","":"downstairs","session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2362025-10-15T16:02:32.108Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.654741279Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"":"downstairs","":"downstairs","session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2372025-10-15T16:02:32.108Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
2382025-10-15T16:02:32.108Z,"time":"2025-10-15T16:02:31.654758528Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2392025-10-15T16:02:32.108Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.654783657Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"client":"0","":"downstairs","session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2402025-10-15T16:02:32.108Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.654800426Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"client":"1","":"downstairs","session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2412025-10-15T16:02:32.108Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.654823465Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"client":"2","":"downstairs","session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2422025-10-15T16:02:32.108Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.654840714Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2432025-10-15T16:02:32.108Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.654863072Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2442025-10-15T16:02:32.108Z{"msg":"0cf2aaa9-507e-4bfa-8d92-45152e3c2633 is now active with session: e82eead9-98bf-4a56-bc3f-bc52b3159e68","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.654881381Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2452025-10-15T16:02:32.108Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.65489697Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246,"session_id":"e82eead9-98bf-4a56-bc3f-bc52b3159e68"}
2462025-10-15T16:02:32.108Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.65491312Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246}
2472025-10-15T16:02:32.108Z{"msg":"Activated sub_volume 0cf2aaa9-507e-4bfa-8d92-45152e3c2633","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:31.654945558Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246}
2482025-10-15T16:02:32.108ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
2492025-10-15T16:02:32.108ZFill test
2502025-10-15T16:02:32.147ZRead and Verify all blocks (0..100 range:false)
2512025-10-15T16:02:32.227ZWrote out file "/var/tmp/test_up-build/verify_file"
2522025-10-15T16:02:32.227ZCLIENT: Tests done. All submitted work has been ACK'd
2532025-10-15T16:02:32.230Z----------------------------------------------------------------
2542025-10-15T16:02:32.230Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2552025-10-15T16:02:32.230ZStates: Active Active Active
2562025-10-15T16:02:32.230ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2572025-10-15T16:02:32.230Z 1002 Acked Read 50 Done Done Done false
2582025-10-15T16:02:32.231Z STATES DS:0 DS:1 DS:2 TOTAL
2592025-10-15T16:02:32.231Z Sent 0 0 0 0
2602025-10-15T16:02:32.231Z Done 1 1 1 3
2612025-10-15T16:02:32.231Z Skipped 0 0 0 0
2622025-10-15T16:02:32.231Z Error 0 0 0 0
2632025-10-15T16:02:32.231ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2642025-10-15T16:02:32.231ZDownstairs last five completed: 1001 1000
2652025-10-15T16:02:32.231ZUpstairs last five completed: 1002 1001 1000
2662025-10-15T16:02:32.231Z----------------------------------------------------------------
2672025-10-15T16:02:32.231Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2682025-10-15T16:02:32.231ZStates: Active Active Active
2692025-10-15T16:02:32.231ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2702025-10-15T16:02:32.231Z 1002 Acked Read 50 Done Done Done false
2712025-10-15T16:02:32.231Z STATES DS:0 DS:1 DS:2 TOTAL
2722025-10-15T16:02:32.231Z Sent 0 0 0 0
2732025-10-15T16:02:32.231Z Done 1 1 1 3
2742025-10-15T16:02:32.231Z Skipped 0 0 0 0
2752025-10-15T16:02:32.231Z Error 0 0 0 0
2762025-10-15T16:02:32.231ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2772025-10-15T16:02:32.231ZDownstairs last five completed: 1001 1000
2782025-10-15T16:02:32.231ZUpstairs last five completed: 1002 1001 1000
2792025-10-15T16:02:32.231ZCLIENT: Up:0 ds:2 act:6
2802025-10-15T16:02:36.228Z----------------------------------------------------------------
2812025-10-15T16:02:36.228Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2822025-10-15T16:02:36.229ZDownstairs last five completed: 1003 1002 1001 1000
2832025-10-15T16:02:36.229ZUpstairs last five completed: 1003 1002 1001 1000
2842025-10-15T16:02:36.232Z----------------------------------------------------------------
2852025-10-15T16:02:36.232Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2862025-10-15T16:02:36.232ZDownstairs last five completed: 1003 1002 1001 1000
2872025-10-15T16:02:36.232ZUpstairs last five completed: 1003 1002 1001 1000
2882025-10-15T16:02:36.232ZCLIENT: Up:0 ds:0 act:6
2892025-10-15T16:02:36.232ZCLIENT: All crucible jobs finished, exiting program
2902025-10-15T16:02:36.232Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-10-15T16:02:35.778420962Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1246}
2912025-10-15T16:02:36.240ZRepair setup passed
2922025-10-15T16:02:36.244ZCopy the region for /var/tmp/test_up-build/8830
2932025-10-15T16:02:36.247Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2942025-10-15T16:02:36.250Z/var/tmp/bins/crutest fill -g 32 -q --key bjJf7udwLUTuGZvKPtAEtzbHDlsPbeDGqK7OCMXKwN4= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2952025-10-15T16:02:36.307Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.855084497Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"task":"crutest"}
2962025-10-15T16:02:36.311Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.855648954Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"task":"crutest"}
2972025-10-15T16:02:36.311Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.857509249Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"task":"crutest"}
2982025-10-15T16:02:36.311Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.857719717Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252}
2992025-10-15T16:02:36.311Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"13b1577b584ca0dbd09b897d333d5c2b128ce008\",\n git_commit_timestamp: \"2025-10-15T15:33:57.000000000Z\",\n git_branch: \"mkeeter/check-hashes\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.857770745Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252}
3002025-10-15T16:02:36.311Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.857795503Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252}
3012025-10-15T16:02:36.311Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.857817952Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252}
3022025-10-15T16:02:36.311Z{"msg":"Crucible b68c18c7-885c-4cea-b55b-be084e280abb has session id: 02b2e130-a672-45ae-b0c6-bcba8e06365e","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.858445287Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3032025-10-15T16:02:36.311Z{"msg":"Upstairs opts: Upstairs UUID: b68c18c7-885c-4cea-b55b-be084e280abb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.858506903Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3042025-10-15T16:02:36.311Z{"msg":"Crucible stats registered with UUID: b68c18c7-885c-4cea-b55b-be084e280abb","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.858546861Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3052025-10-15T16:02:36.311Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-10-15T16:02:35.858569699Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3062025-10-15T16:02:36.311Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.858656564Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3072025-10-15T16:02:36.311Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.858706242Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3082025-10-15T16:02:36.311Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.858745279Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3092025-10-15T16:02:36.315Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.860660471Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"task":"crutest"}
3102025-10-15T16:02:36.315Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.860736857Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252}
3112025-10-15T16:02:36.315Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"13b1577b584ca0dbd09b897d333d5c2b128ce008\",\n git_commit_timestamp: \"2025-10-15T15:33:57.000000000Z\",\n git_branch: \"mkeeter/check-hashes\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.860809423Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252}
3122025-10-15T16:02:36.315Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.860845081Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252}
3132025-10-15T16:02:36.315Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.860881089Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252}
3142025-10-15T16:02:36.315Z{"msg":"Crucible b68c18c7-885c-4cea-b55b-be084e280abb has session id: 40ea8cb1-5d82-4cc2-8af8-625686b0736d","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.861306774Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3152025-10-15T16:02:36.315Z{"msg":"Upstairs opts: Upstairs UUID: b68c18c7-885c-4cea-b55b-be084e280abb, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.861368041Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3162025-10-15T16:02:36.315Z{"msg":"Crucible stats registered with UUID: b68c18c7-885c-4cea-b55b-be084e280abb","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.861401009Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3172025-10-15T16:02:36.315Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-10-15T16:02:35.861460476Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3182025-10-15T16:02:36.315Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.861489244Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3192025-10-15T16:02:36.315Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.861529202Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3202025-10-15T16:02:36.315Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.861549871Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3212025-10-15T16:02:36.315Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.862281219Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"task":"crutest"}
3222025-10-15T16:02:36.315Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.862332747Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252}
3232025-10-15T16:02:36.315Z{"msg":"b68c18c7-885c-4cea-b55b-be084e280abb active request set","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.862415532Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3242025-10-15T16:02:36.315Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.86245103Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"client":"0","":"downstairs","session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3252025-10-15T16:02:36.315Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.862470199Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"client":"1","":"downstairs","session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3262025-10-15T16:02:36.315Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.862488098Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"client":"2","":"downstairs","session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3272025-10-15T16:02:36.315Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.862520456Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3282025-10-15T16:02:36.315Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.862546035Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3292025-10-15T16:02:36.315Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.862580813Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3302025-10-15T16:02:36.315Z{"msg":"ds_connection connected from Ok(127.0.0.1:63960)","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.862608791Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3312025-10-15T16:02:36.315Z{"msg":"ds_connection connected from Ok(127.0.0.1:47453)","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.862648039Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3322025-10-15T16:02:36.315Z{"msg":"ds_connection connected from Ok(127.0.0.1:59338)","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.862934773Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3332025-10-15T16:02:36.316Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.863982123Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"client":"1","":"downstairs","session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3342025-10-15T16:02:36.316Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.864117136Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"client":"2","":"downstairs","session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3352025-10-15T16:02:36.316Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.864160903Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"client":"0","":"downstairs","session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3362025-10-15T16:02:36.316Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.864384651Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3372025-10-15T16:02:36.321Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.864507304Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3382025-10-15T16:02:36.321Z{"msg":"[0]R flush_numbers: [4002, 4002, 4002, 4002, 4002]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.864557401Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"downstairs","":"downstairs","session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3392025-10-15T16:02:36.321Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.864599978Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"downstairs","":"downstairs","session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3402025-10-15T16:02:36.321Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.864640866Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"downstairs","":"downstairs","session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3412025-10-15T16:02:36.321Z{"msg":"[1]R flush_numbers: [4002, 4002, 4002, 4002, 4002]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.864682074Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"downstairs","":"downstairs","session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3422025-10-15T16:02:36.321Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.864722401Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"downstairs","":"downstairs","session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3432025-10-15T16:02:36.321Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.864765939Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"downstairs","":"downstairs","session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3442025-10-15T16:02:36.321Z{"msg":"[2]R flush_numbers: [4002, 4002, 4002, 4002, 4002]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.864809707Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"downstairs","":"downstairs","session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3452025-10-15T16:02:36.321Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.864856744Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"downstairs","":"downstairs","session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3462025-10-15T16:02:36.321Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.864905421Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"downstairs","":"downstairs","session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3472025-10-15T16:02:36.321Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.864946259Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"downstairs","":"downstairs","session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3482025-10-15T16:02:36.321Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.864979607Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"downstairs","":"downstairs","session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3492025-10-15T16:02:36.321Z{"msg":"Next flush: 4003","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.865234923Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"downstairs","":"downstairs","session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3502025-10-15T16:02:36.321Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.865302149Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"downstairs","":"downstairs","session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3512025-10-15T16:02:36.321Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.865343716Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3522025-10-15T16:02:36.321Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.865376584Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"client":"0","":"downstairs","session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3532025-10-15T16:02:36.321Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.865413112Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"client":"1","":"downstairs","session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3542025-10-15T16:02:36.321Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.865538785Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"client":"2","":"downstairs","session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3552025-10-15T16:02:36.321Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.865598492Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3562025-10-15T16:02:36.321Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.865647509Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3572025-10-15T16:02:36.321Z{"msg":"b68c18c7-885c-4cea-b55b-be084e280abb is now active with session: 02b2e130-a672-45ae-b0c6-bcba8e06365e","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.865692106Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3582025-10-15T16:02:36.321Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.865739114Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"session_id":"02b2e130-a672-45ae-b0c6-bcba8e06365e"}
3592025-10-15T16:02:36.321Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.865770252Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252}
3602025-10-15T16:02:36.321Z{"msg":"Activated sub_volume b68c18c7-885c-4cea-b55b-be084e280abb","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.86580525Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252}
3612025-10-15T16:02:36.321Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.865848088Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252}
3622025-10-15T16:02:36.321Z{"msg":"b68c18c7-885c-4cea-b55b-be084e280abb active request set","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.865873386Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3632025-10-15T16:02:36.321Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.865894535Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"client":"0","":"downstairs","session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3642025-10-15T16:02:36.321Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.865921533Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"client":"1","":"downstairs","session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3652025-10-15T16:02:36.321Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.865951282Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"client":"2","":"downstairs","session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3662025-10-15T16:02:36.321Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.86598584Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3672025-10-15T16:02:36.321Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.866041557Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3682025-10-15T16:02:36.321Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.866080195Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3692025-10-15T16:02:36.321Z{"msg":"ds_connection connected from Ok(127.0.0.1:46559)","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.866126232Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3702025-10-15T16:02:36.321Z{"msg":"ds_connection connected from Ok(127.0.0.1:57264)","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.866179459Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3712025-10-15T16:02:36.321Z{"msg":"ds_connection connected from Ok(127.0.0.1:44539)","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.866220157Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3722025-10-15T16:02:36.322Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.866242666Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"client":"2","":"downstairs","session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3732025-10-15T16:02:36.322Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.866270134Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"client":"1","":"downstairs","session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3742025-10-15T16:02:36.322Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.866294312Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"client":"0","":"downstairs","session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3752025-10-15T16:02:36.322Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.866533679Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3762025-10-15T16:02:36.322Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.866633143Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3772025-10-15T16:02:36.322Z{"msg":"[0]R flush_numbers: [5326, 5326, 5326, 5326, 5326]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.866670831Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"downstairs","":"downstairs","session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3782025-10-15T16:02:36.322Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.86668923Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"downstairs","":"downstairs","session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3792025-10-15T16:02:36.322Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.866714039Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"downstairs","":"downstairs","session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3802025-10-15T16:02:36.322Z{"msg":"[1]R flush_numbers: [5326, 5326, 5326, 5326, 5326]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.866732088Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"downstairs","":"downstairs","session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3812025-10-15T16:02:36.322Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.866756016Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"downstairs","":"downstairs","session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3822025-10-15T16:02:36.322Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.866773575Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"downstairs","":"downstairs","session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3832025-10-15T16:02:36.322Z{"msg":"[2]R flush_numbers: [5326, 5326, 5326, 5326, 5326]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.866790704Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"downstairs","":"downstairs","session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3842025-10-15T16:02:36.322Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"Wait for a query_work_queue command to finish before sending IO
3852025-10-15T16:02:36.322Zcrucible","level":30,"time":"2025-10-15T16:02:35.866821613Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"downstairs","":"downstairs","session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3862025-10-15T16:02:36.322Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.866839282Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"downstairs","":"downstairs","session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3872025-10-15T16:02:36.322Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.86686199Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"downstairs","":"downstairs","session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3882025-10-15T16:02:36.322Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.866879059Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"downstairs","":"downstairs","session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3892025-10-15T16:02:36.322Z{"msg":"Next flush: 5327","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.866901358Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"downstairs","":"downstairs","session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3902025-10-15T16:02:36.322Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.866917657Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"":"downstairs","":"downstairs","session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3912025-10-15T16:02:36.322Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.866944946Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3922025-10-15T16:02:36.322Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.866963865Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"client":"0","":"downstairs","session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3932025-10-15T16:02:36.322Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.866991883Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"client":"1","":"downstairs","session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3942025-10-15T16:02:36.322Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.867016892Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"client":"2","":"downstairs","session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3952025-10-15T16:02:36.322Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.8670564Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3962025-10-15T16:02:36.322Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.867088388Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3972025-10-15T16:02:36.322Z{"msg":"b68c18c7-885c-4cea-b55b-be084e280abb is now active with session: 40ea8cb1-5d82-4cc2-8af8-625686b0736d","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.867120396Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3982025-10-15T16:02:36.322Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.867158664Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252,"session_id":"40ea8cb1-5d82-4cc2-8af8-625686b0736d"}
3992025-10-15T16:02:36.322Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.867188082Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252}
4002025-10-15T16:02:36.322Z{"msg":"Activated sub_volume b68c18c7-885c-4cea-b55b-be084e280abb","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:35.867208481Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252}
4012025-10-15T16:02:36.322ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
4022025-10-15T16:02:36.322ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4032025-10-15T16:02:36.322ZFill test
4042025-10-15T16:02:36.361ZRead and Verify all blocks (0..100 range:false)
4052025-10-15T16:02:36.441ZWrote out file "/var/tmp/test_up-build/verify_file"
4062025-10-15T16:02:36.441ZCLIENT: Tests done. All submitted work has been ACK'd
4072025-10-15T16:02:36.441Z----------------------------------------------------------------
4082025-10-15T16:02:36.441Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4092025-10-15T16:02:36.441ZStates: Active Active Active
4102025-10-15T16:02:36.441ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4112025-10-15T16:02:36.444Z 1002 Acked Read 50 Done Done Done false
4122025-10-15T16:02:36.444Z STATES DS:0 DS:1 DS:2 TOTAL
4132025-10-15T16:02:36.444Z Sent 0 0 0 0
4142025-10-15T16:02:36.444Z Done 1 1 1 3
4152025-10-15T16:02:36.445Z Skipped 0 0 0 0
4162025-10-15T16:02:36.445Z Error 0 0 0 0
4172025-10-15T16:02:36.445ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4182025-10-15T16:02:36.445ZDownstairs last five completed: 1001 1000
4192025-10-15T16:02:36.445ZUpstairs last five completed: 1002 1001 1000
4202025-10-15T16:02:36.445Z----------------------------------------------------------------
4212025-10-15T16:02:36.445Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4222025-10-15T16:02:36.445ZStates: Active Active Active
4232025-10-15T16:02:36.445ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4242025-10-15T16:02:36.445Z 1002 Acked Read 50 Done Done Done false
4252025-10-15T16:02:36.445Z STATES DS:0 DS:1 DS:2 TOTAL
4262025-10-15T16:02:36.445Z Sent 0 0 0 0
4272025-10-15T16:02:36.445Z Done 1 1 1 3
4282025-10-15T16:02:36.445Z Skipped 0 0 0 0
4292025-10-15T16:02:36.445Z Error 0 0 0 0
4302025-10-15T16:02:36.445ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4312025-10-15T16:02:36.445ZDownstairs last five completed: 1001 1000
4322025-10-15T16:02:36.445ZUpstairs last five completed: 1002 1001 1000
4332025-10-15T16:02:36.445ZCLIENT: Up:0 ds:2 act:6
4342025-10-15T16:02:40.441Z----------------------------------------------------------------
4352025-10-15T16:02:40.441Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4362025-10-15T16:02:40.441ZDownstairs last five completed: 1003 1002 1001 1000
4372025-10-15T16:02:40.441ZUpstairs last five completed: 1003 1002 1001 1000
4382025-10-15T16:02:40.441Z----------------------------------------------------------------
4392025-10-15T16:02:40.441Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4402025-10-15T16:02:40.441ZDownstairs last five completed: 1003 1002 1001 1000
4412025-10-15T16:02:40.441ZUpstairs last five completed: 1003 1002 1001 1000
4422025-10-15T16:02:40.445ZCLIENT: Up:0 ds:0 act:6
4432025-10-15T16:02:40.445ZCLIENT: All crucible jobs finished, exiting program
4442025-10-15T16:02:40.445Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-10-15T16:02:39.990428501Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1252}
4452025-10-15T16:02:40.448ZRepair part 1 passed
4462025-10-15T16:02:40.448Z
4472025-10-15T16:02:40.448ZKill the current downstairs
4482025-10-15T16:02:40.625ZDownstairs 2 stopped
4492025-10-15T16:02:40.625Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
4502025-10-15T16:02:40.629ZNow put back the original so we have a mismatch
4512025-10-15T16:02:40.629Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
4522025-10-15T16:02:40.632ZRestart downstairs with old directory
4532025-10-15T16:02:40.688Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4542025-10-15T16:02:40.707ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4552025-10-15T16:02:40.710Z 0 000-009 32 32 31 4003 4003 4002 F F F <---
4562025-10-15T16:02:40.710Z 1 010-019 32 32 31 4003 4003 4002 F F F <---
4572025-10-15T16:02:40.710Z 2 020-029 32 32 31 4003 4003 4002 F F F <---
4582025-10-15T16:02:40.710Z 3 030-039 32 32 31 4003 4003 4002 F F F <---
4592025-10-15T16:02:40.710Z 4 040-049 32 32 31 4003 4003 4002 F F F <---
4602025-10-15T16:02:40.710ZMax gen: 32, Max flush: 4003
4612025-10-15T16:02:40.710ZError: Difference in extent metadata found!
4622025-10-15T16:02:40.714Zdump test found error as expected
4632025-10-15T16:02:40.714Z
4642025-10-15T16:02:40.714Z
4652025-10-15T16:02:40.714Z/var/tmp/bins/crutest verify --range -g 33 -q --key bjJf7udwLUTuGZvKPtAEtzbHDlsPbeDGqK7OCMXKwN4= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
4662025-10-15T16:02:40.771Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.319837112Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"task":"crutest"}
4672025-10-15T16:02:40.775Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.320450097Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"task":"crutest"}
4682025-10-15T16:02:40.775Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.322424275Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"task":"crutest"}
4692025-10-15T16:02:40.775Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.322647342Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260}
4702025-10-15T16:02:40.775Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"13b1577b584ca0dbd09b897d333d5c2b128ce008\",\n git_commit_timestamp: \"2025-10-15T15:33:57.000000000Z\",\n git_branch: \"mkeeter/check-hashes\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.322709809Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260}
4712025-10-15T16:02:40.775Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.322739287Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260}
4722025-10-15T16:02:40.775Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.322767716Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260}
4732025-10-15T16:02:40.775Z{"msg":"Crucible bb959ed9-52b2-405a-a5de-33d96f593764 has session id: 5610221a-a4cf-476e-b3a2-a9f9d615ecd8","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.323456297Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
4742025-10-15T16:02:40.775Z{"msg":"Upstairs opts: Upstairs UUID: bb959ed9-52b2-405a-a5de-33d96f593764, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.323548582Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
4752025-10-15T16:02:40.775Z{"msg":"Crucible stats registered with UUID: bb959ed9-52b2-405a-a5de-33d96f593764","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.323626837Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
4762025-10-15T16:02:40.775Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-10-15T16:02:40.323677974Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
4772025-10-15T16:02:40.775Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.323702223Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"io task","client":"1","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
4782025-10-15T16:02:40.775Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.323743531Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"io task","client":"0","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
4792025-10-15T16:02:40.775Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.323763579Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
4802025-10-15T16:02:40.779Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.325741027Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"task":"crutest"}
4812025-10-15T16:02:40.779Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.325845522Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260}
4822025-10-15T16:02:40.779Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"13b1577b584ca0dbd09b897d333d5c2b128ce008\",\n git_commit_timestamp: \"2025-10-15T15:33:57.000000000Z\",\n git_branch: \"mkeeter/check-hashes\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.325893029Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260}
4832025-10-15T16:02:40.779Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.325916088Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260}
4842025-10-15T16:02:40.779Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.325938976Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260}
4852025-10-15T16:02:40.779Z{"msg":"Crucible bb959ed9-52b2-405a-a5de-33d96f593764 has session id: 399682ca-89c3-4e7d-bc44-0e9d4cf9c24a","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.32639793Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
4862025-10-15T16:02:40.779Z{"msg":"Upstairs opts: Upstairs UUID: bb959ed9-52b2-405a-a5de-33d96f593764, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.326460767Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
4872025-10-15T16:02:40.779Z{"msg":"Crucible stats registered with UUID: bb959ed9-52b2-405a-a5de-33d96f593764","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.326511984Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
4882025-10-15T16:02:40.779Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-10-15T16:02:40.326550822Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
4892025-10-15T16:02:40.779Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.32657746Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"io task","client":"0","":"downstairs","session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
4902025-10-15T16:02:40.779Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.326611778Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"io task","client":"1","":"downstairs","session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
4912025-10-15T16:02:40.779Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.326649516Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
4922025-10-15T16:02:40.779Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.327523107Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"task":"crutest"}
4932025-10-15T16:02:40.779Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.327576234Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260}
4942025-10-15T16:02:40.779Z{"msg":"bb959ed9-52b2-405a-a5de-33d96f593764 active request set","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.327624441Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
4952025-10-15T16:02:40.779Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.327665569Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"client":"0","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
4962025-10-15T16:02:40.779Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.327699417Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"client":"1","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
4972025-10-15T16:02:40.779Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.327728455Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"client":"2","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
4982025-10-15T16:02:40.779Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.327768493Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
4992025-10-15T16:02:40.779Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.327808841Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"io task","client":"0","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5002025-10-15T16:02:40.783Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.327848408Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"io task","client":"1","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5012025-10-15T16:02:40.783Z{"msg":"ds_connection connected from Ok(127.0.0.1:34029)","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.327896626Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"io task","client":"0","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5022025-10-15T16:02:40.783Z{"msg":"ds_connection connected from Ok(127.0.0.1:53452)","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.327935363Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"io task","client":"1","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5032025-10-15T16:02:40.783Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-10-15T16:02:40.327986331Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5042025-10-15T16:02:40.783Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-10-15T16:02:40.328110784Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5052025-10-15T16:02:40.783Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-10-15T16:02:40.328153571Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5062025-10-15T16:02:40.783Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.328261105Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5072025-10-15T16:02:40.783Z{"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.328311272Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5082025-10-15T16:02:40.783Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.329038821Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"client":"1","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5092025-10-15T16:02:40.783Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.329160254Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"client":"0","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5102025-10-15T16:02:40.783Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.329441548Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5112025-10-15T16:02:40.783Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:40.329523074Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5122025-10-15T16:02:50.782Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.329949106Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5132025-10-15T16:02:50.782Z{"msg":"ds_connection connected from Ok(127.0.0.1:42324)","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.33004146Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5142025-10-15T16:02:50.787Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.33146777Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"client":"2","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5152025-10-15T16:02:50.787Z{"msg":"[0]R flush_numbers: [4003, 4003, 4003, 4003, 4003]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.331862227Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"downstairs","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5162025-10-15T16:02:50.787Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.331907185Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"downstairs","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5172025-10-15T16:02:50.787Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.331948602Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"downstairs","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5182025-10-15T16:02:50.787Z{"msg":"[1]R flush_numbers: [4003, 4003, 4003, 4003, 4003]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.33198648Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"downstairs","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5192025-10-15T16:02:50.787Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.332025138Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"downstairs","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5202025-10-15T16:02:50.787Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.332061966Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"downstairs","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5212025-10-15T16:02:50.787Z{"msg":"[2]R flush_numbers: [4002, 4002, 4002, 4002, 4002]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.332107824Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"downstairs","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5222025-10-15T16:02:50.787Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.332154731Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"downstairs","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5232025-10-15T16:02:50.787Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.332203648Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"downstairs","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5242025-10-15T16:02:50.787Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.332261205Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"downstairs","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5252025-10-15T16:02:50.787Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.332317282Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"downstairs","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5262025-10-15T16:02:50.787Z{"msg":"Next flush: 4004","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.332382618Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"downstairs","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5272025-10-15T16:02:50.787Z{"msg":"extent 0 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { gen: 32, flush: 4003, dirty: false }), Some(ExtentMetadata { gen: 32, flush: 4003, dirty: false }), Some(ExtentMetadata { gen: 31, flush: 4002, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.332432655Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5282025-10-15T16:02:50.787Z{"msg":"extent:0 ClientMap(ClientData([Some(ExtentMetadata { gen: 32, flush: 4003, dirty: false }), Some(ExtentMetadata { gen: 32, flush: 4003, dirty: false }), Some(ExtentMetadata { gen: 31, flush: 4002, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.332473143Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5292025-10-15T16:02:50.787Z{"msg":"found dest for source 0 for extent at index 0 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.332510571Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5302025-10-15T16:02:50.787Z{"msg":"extent 1 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { gen: 32, flush: 4003, dirty: false }), Some(ExtentMetadata { gen: 32, flush: 4003, dirty: false }), Some(ExtentMetadata { gen: 31, flush: 4002, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.332579657Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5312025-10-15T16:02:50.788Z{"msg":"extent:1 ClientMap(ClientData([Some(ExtentMetadata { gen: 32, flush: 4003, dirty: false }), Some(ExtentMetadata { gen: 32, flush: 4003, dirty: false }), Some(ExtentMetadata { gen: 31, flush: 4002, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.332655562Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5322025-10-15T16:02:50.788Z{"msg":"found dest for source 0 for extent at index 1 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.332722729Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5332025-10-15T16:02:50.788Z{"msg":"extent 2 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { gen: 32, flush: 4003, dirty: false }), Some(ExtentMetadata { gen: 32, flush: 4003, dirty: false }), Some(ExtentMetadata { gen: 31, flush: 4002, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.332774806Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5342025-10-15T16:02:50.788Z{"msg":"extent:2 ClientMap(ClientData([Some(ExtentMetadata { gen: 32, flush: 4003, dirty: false }), Some(ExtentMetadata { gen: 32, flush: 4003, dirty: false }), Some(ExtentMetadata { gen: 31, flush: 4002, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.332823253Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5352025-10-15T16:02:50.788Z{"msg":"found dest for source 0 for extent at index 2 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.332856541Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5362025-10-15T16:02:50.788Z{"msg":"extent 3 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { gen: 32, flush: 4003, dirty: false }), Some(ExtentMetadata { gen: 32, flush: 4003, dirty: false }), Some(ExtentMetadata { gen: 31, flush: 4002, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.33288534Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5372025-10-15T16:02:50.788Z{"msg":"extent:3 ClientMap(ClientData([Some(ExtentMetadata { gen: 32, flush: 4003, dirty: false }), Some(ExtentMetadata { gen: 32, flush: 4003, dirty: false }), Some(ExtentMetadata { gen: 31, flush: 4002, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.332916548Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5382025-10-15T16:02:50.788Z{"msg":"found dest for source 0 for extent at index 3 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.332935507Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5392025-10-15T16:02:50.788Z{"msg":"extent 4 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { gen: 32, flush: 4003, dirty: false }), Some(ExtentMetadata { gen: 32, flush: 4003, dirty: false }), Some(ExtentMetadata { gen: 31, flush: 4002, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.332966745Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5402025-10-15T16:02:50.788Z{"msg":"extent:4 ClientMap(ClientData([Some(ExtentMetadata { gen: 32, flush: 4003, dirty: false }), Some(ExtentMetadata { gen: 32, flush: 4003, dirty: false }), Some(ExtentMetadata { gen: 31, flush: 4002, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.332991203Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5412025-10-15T16:02:50.788Z{"msg":"found dest for source 0 for extent at index 4 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.333016162Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5422025-10-15T16:02:50.788Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.33304727Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"client":"0","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5432025-10-15T16:02:50.788Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.333073939Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"client":"1","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5442025-10-15T16:02:50.788Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.333092328Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"client":"2","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5452025-10-15T16:02:50.788Z{"msg":"Full repair list: {ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.333128936Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"downstairs","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5462025-10-15T16:02:50.788Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 4004, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 4004, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 4004, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 4004, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 4004, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.333355273Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"downstairs","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5472025-10-15T16:02:50.788Z{"msg":"starting reconciliation b474032d-8d93-451a-bbb9-21cfe060f0bb: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.333391541Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"downstairs","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5482025-10-15T16:02:50.788Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.33340979Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"client":"2","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5492025-10-15T16:02:50.795Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.343538367Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"client":"2","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5502025-10-15T16:02:50.799Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.347396979Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"client":"2","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5512025-10-15T16:02:50.802Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.351109249Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"client":"2","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5522025-10-15T16:02:50.806Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.354889995Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"client":"2","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5532025-10-15T16:02:50.809Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.358130202Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"downstairs","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5542025-10-15T16:02:50.809Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.358170759Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"client":"0","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5552025-10-15T16:02:50.813Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.358192658Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"client":"1","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5562025-10-15T16:02:50.813Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.358229706Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"client":"2","":"downstairs","session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5572025-10-15T16:02:50.813Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.358273434Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5582025-10-15T16:02:50.813Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.358306272Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5592025-10-15T16:02:50.813Z{"msg":"bb959ed9-52b2-405a-a5de-33d96f593764 is now active with session: 5610221a-a4cf-476e-b3a2-a9f9d615ecd8","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.35833359Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"session_id":"5610221a-a4cf-476e-b3a2-a9f9d615ecd8"}
5602025-10-15T16:02:50.813Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.358360709Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260}
5612025-10-15T16:02:50.813Z{"msg":"Activated sub_volume bb959ed9-52b2-405a-a5de-33d96f593764","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.358389517Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260}
5622025-10-15T16:02:50.813Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.358404876Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260}
5632025-10-15T16:02:50.813Z{"msg":"bb959ed9-52b2-405a-a5de-33d96f593764 active request set","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.358425535Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5642025-10-15T16:02:50.813Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.358454354Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"client":"0","":"downstairs","session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5652025-10-15T16:02:50.813Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.358484252Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"client":"1","":"downstairs","session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5662025-10-15T16:02:50.813Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.35851209Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"client":"2","":"downstairs","session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5672025-10-15T16:02:50.813Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.358544009Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5682025-10-15T16:02:50.813Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.358578927Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"io task","client":"0","":"downstairs","session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5692025-10-15T16:02:50.814Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.358611905Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"io task","client":"1","":"downstairs","session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5702025-10-15T16:02:50.814Z{"msg":"ds_connection connected from Ok(127.0.0.1:60504)","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.358646083Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5712025-10-15T16:02:50.814Z{"msg":"ds_connection connected from Ok(127.0.0.1:55283)","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.35868431Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"io task","client":"1","":"downstairs","session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5722025-10-15T16:02:50.814Z{"msg":"ds_connection connected from Ok(127.0.0.1:40860)","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.358720279Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"io task","client":"0","":"downstairs","session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5732025-10-15T16:02:50.814Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.359477406Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"client":"1","":"downstairs","session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5742025-10-15T16:02:50.814Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.359593449Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"client":"0","":"downstairs","session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5752025-10-15T16:02:50.814Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.359785158Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"client":"2","":"downstairs","session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5762025-10-15T16:02:50.814Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.35993782Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5772025-10-15T16:02:50.814Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.359982607Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5782025-10-15T16:02:50.814Z{"msg":"[0]R flush_numbers: [5327, 5327, 5327, 5327, 5327]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.360155227Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"downstairs","":"downstairs","session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5792025-10-15T16:02:50.814Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.360194285Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"downstairs","":"downstairs","session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5802025-10-15T16:02:50.814Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.360213844Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"downstairs","":"downstairs","session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5812025-10-15T16:02:50.814Z{"msg":"[1]R flush_numbers: [5327, 5327, 5327, 5327, 5327]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.360232293Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"downstairs","":"downstairs","session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5822025-10-15T16:02:50.814Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.360265781Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"downstairs","":"downstairs","session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5832025-10-15T16:02:50.814Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.360292679Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"downstairs","":"downstairs","session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5842025-10-15T16:02:50.814Z{"msg":"[2]R flush_numbers: [5327, 5327, 5327, 5327, 5327]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
5852025-10-15T16:02:50.814Z,"time":"2025-10-15T16:02:50.360322598Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"downstairs","":"downstairs","session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5862025-10-15T16:02:50.814Z{"msg":"[2]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.360367305Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"downstairs","":"downstairs","session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5872025-10-15T16:02:50.814Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.360393544Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"downstairs","":"downstairs","session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5882025-10-15T16:02:50.814Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.360416332Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"downstairs","":"downstairs","session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5892025-10-15T16:02:50.814Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.360437181Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"downstairs","":"downstairs","session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5902025-10-15T16:02:50.814Z{"msg":"Next flush: 5328","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.36046378Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"downstairs","":"downstairs","session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5912025-10-15T16:02:50.814Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.360491378Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"":"downstairs","":"downstairs","session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5922025-10-15T16:02:50.814Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.360510467Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5932025-10-15T16:02:50.814Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.360537126Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"client":"0","":"downstairs","session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5942025-10-15T16:02:50.814Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.360567684Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"client":"1","":"downstairs","session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5952025-10-15T16:02:50.814Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.360591623Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"client":"2","":"downstairs","session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5962025-10-15T16:02:50.814Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.360611841Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5972025-10-15T16:02:50.814Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.36063817Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5982025-10-15T16:02:50.814Z{"msg":"bb959ed9-52b2-405a-a5de-33d96f593764 is now active with session: 399682ca-89c3-4e7d-bc44-0e9d4cf9c24a","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.360661339Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
5992025-10-15T16:02:50.814Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.360685697Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260,"session_id":"399682ca-89c3-4e7d-bc44-0e9d4cf9c24a"}
6002025-10-15T16:02:50.814Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.360721565Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260}
6012025-10-15T16:02:50.814Z{"msg":"Activated sub_volume bb959ed9-52b2-405a-a5de-33d96f593764","v":0,"name":"crucible","level":30,"time":"2025-10-15T16:02:50.360746124Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260}
6022025-10-15T16:02:50.814ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
6032025-10-15T16:02:50.814ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
6042025-10-15T16:02:50.814ZRead and Verify all blocks (0..100 range:true)
6052025-10-15T16:02:50.896ZWrote out file "/var/tmp/test_up-build/verify_file"
6062025-10-15T16:02:50.896ZVerify test completed
6072025-10-15T16:02:50.896ZWrote out file "/var/tmp/test_up-build/verify_file"
6082025-10-15T16:02:50.896ZCLIENT: Tests done. All submitted work has been ACK'd
6092025-10-15T16:02:50.900Z----------------------------------------------------------------
6102025-10-15T16:02:50.901Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6112025-10-15T16:02:50.901ZStates: Active Active Active
6122025-10-15T16:02:50.901ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6132025-10-15T16:02:50.901Z 1000 Acked Read 50 Done Done Done false
6142025-10-15T16:02:50.901Z STATES DS:0 DS:1 DS:2 TOTAL
6152025-10-15T16:02:50.901Z Sent 0 0 0 0
6162025-10-15T16:02:50.901Z Done 1 1 1 3
6172025-10-15T16:02:50.901Z Skipped 0 0 0 0
6182025-10-15T16:02:50.901Z Error 0 0 0 0
6192025-10-15T16:02:50.901ZLast Flush: None None None
6202025-10-15T16:02:50.901ZDownstairs last five completed:
6212025-10-15T16:02:50.901ZUpstairs last five completed: 1000
6222025-10-15T16:02:50.901Z----------------------------------------------------------------
6232025-10-15T16:02:50.901Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6242025-10-15T16:02:50.901ZStates: Active Active Active
6252025-10-15T16:02:50.901ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6262025-10-15T16:02:50.901Z 1000 Acked Read 50 Done Done Sent false
6272025-10-15T16:02:50.901Z STATES DS:0 DS:1 DS:2 TOTAL
6282025-10-15T16:02:50.901Z Sent 0 0 1 1
6292025-10-15T16:02:50.901Z Done 1 1 0 2
6302025-10-15T16:02:50.901Z Skipped 0 0 0 0
6312025-10-15T16:02:50.901Z Error 0 0 0 0
6322025-10-15T16:02:50.901ZLast Flush: None None None
6332025-10-15T16:02:50.901ZDownstairs last five completed:
6342025-10-15T16:02:50.901ZUpstairs last five completed: 1000
6352025-10-15T16:02:50.901ZCLIENT: Up:0 ds:2 act:6
6362025-10-15T16:02:54.899Z----------------------------------------------------------------
6372025-10-15T16:02:54.899Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6382025-10-15T16:02:54.899ZDownstairs last five completed: 1001 1000
6392025-10-15T16:02:54.899ZUpstairs last five completed: 1001 1000
6402025-10-15T16:02:54.899Z----------------------------------------------------------------
6412025-10-15T16:02:54.899Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6422025-10-15T16:02:54.899ZDownstairs last five completed: 1001 1000
6432025-10-15T16:02:54.899ZUpstairs last five completed: 1001 1000
6442025-10-15T16:02:54.902ZCLIENT: Up:0 ds:0 act:6
6452025-10-15T16:02:54.902ZCLIENT: All crucible jobs finished, exiting program
6462025-10-15T16:02:54.902Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-10-15T16:02:54.446671793Z","hostname":"w-01K7M9AKW94XMDZXJKNDC1H832","pid":1260}
6472025-10-15T16:02:54.905ZRepair part 2 passed
6482025-10-15T16:02:54.908Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
6492025-10-15T16:02:54.924ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6502025-10-15T16:02:54.924Z 0 000-009 32 32 32 4003 4003 4003 F F F
6512025-10-15T16:02:54.924Z 1 010-019 32 32 32 4003 4003 4003 F F F
6522025-10-15T16:02:54.924Z 2 020-029 32 32 32 4003 4003 4003 F F F
6532025-10-15T16:02:54.924Z 3 030-039 32 32 32 4003 4003 4003 F F F
6542025-10-15T16:02:54.924Z 4 040-049 32 32 32 4003 4003 4003 F F F
6552025-10-15T16:02:54.927ZMax gen: 32, Max flush: 4003
6562025-10-15T16:02:54.930Zdump test passed
6572025-10-15T16:02:54.930Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
6582025-10-15T16:02:54.945Z Extent 1
6592025-10-15T16:02:54.949ZGEN 32 32 32
6602025-10-15T16:02:54.949ZFLUSH_ID 4003 4003 4003
6612025-10-15T16:02:54.949ZDIRTY
6622025-10-15T16:02:54.949Z
6632025-10-15T16:02:54.949ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
6642025-10-15T16:02:54.949Z010 A A A A A A
6652025-10-15T16:02:54.952Z011 A A A A A A
6662025-10-15T16:02:54.952Z012 A A A A A A
6672025-10-15T16:02:54.955Z013 A A A A A A
6682025-10-15T16:02:54.955Z014 A A A A A A
6692025-10-15T16:02:54.955Z015 A A A A A A
6702025-10-15T16:02:54.958Z016 A A A A A A
6712025-10-15T16:02:54.958Z017 A A A A A A
6722025-10-15T16:02:54.961Z018 A A A A A A
6732025-10-15T16:02:54.961Z019 A A A A A A
6742025-10-15T16:02:54.965Zdump extent test passed
6752025-10-15T16:02:54.969Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
6762025-10-15T16:02:54.983ZExtent 2 Block in extent 0 Actual block 20
6772025-10-15T16:02:54.987Z
6782025-10-15T16:02:54.987Z DATA SHA256 VER
6792025-10-15T16:02:54.987Z------ ---------------------------------------------------------------- ---
6802025-10-15T16:02:54.987Z 0 af33f2f0255e0016a35e76c498df58c0fe136a5c5bdc535fdc7f42780a1178b2 A
6812025-10-15T16:02:54.987Z 1 af33f2f0255e0016a35e76c498df58c0fe136a5c5bdc535fdc7f42780a1178b2 A
6822025-10-15T16:02:54.987Z 2 af33f2f0255e0016a35e76c498df58c0fe136a5c5bdc535fdc7f42780a1178b2 A
6832025-10-15T16:02:54.987Z
6842025-10-15T16:02:54.987ZNONCES 0 1 2 DIFF
6852025-10-15T16:02:54.987Z------ ------------------------ ------------------------ ------------------------ -----
6862025-10-15T16:02:54.987Z 0 2a66e899ff9cc7aeb5af796d 2a66e899ff9cc7aeb5af796d 2a66e899ff9cc7aeb5af796d
6872025-10-15T16:02:54.987Z
6882025-10-15T16:02:54.987Z TAGS 0 1 2 DIFF
6892025-10-15T16:02:54.987Z------ -------------------------------- -------------------------------- -------------------------------- -----
6902025-10-15T16:02:54.987Z 0 1c76dee19689d88f2ccafab307520226 1c76dee19689d88f2ccafab307520226 1c76dee19689d88f2ccafab307520226
6912025-10-15T16:02:54.987Z
6922025-10-15T16:02:54.987ZHASHES 0 1 2 DIFF
6932025-10-15T16:02:54.987Z------ ---------------- ---------------- ---------------- -----
6942025-10-15T16:02:54.987Z
6952025-10-15T16:02:54.993Zdump block test passed
6962025-10-15T16:02:54.996ZInitial upstairs tests have completed, stopping all downstairs
6972025-10-15T16:03:00.059ZCreating 7 larger downstairs regions
6982025-10-15T16:03:00.430ZStarting 7 downstairs
6992025-10-15T16:03:05.439Zdsc restarted at PID: 1276
7002025-10-15T16:03:05.862ZNow do the replace-reconcile test
7012025-10-15T16:03:05.921ZUsing 8870 for the replacement port
7022025-10-15T16:06:57.353ZCompleted test: replace-reconcile
7032025-10-15T16:06:57.353ZNow do the replace-before-active test
7042025-10-15T16:13:03.072ZCompleted test: replace-before-active
7052025-10-15T16:13:03.072ZAll tests have completed, stopping all downstairs
7062025-10-15T16:13:03.131Z
7072025-10-15T16:13:03.131ZAll Tests have passed
7082025-10-15T16:13:03.135Z16:17 Test duration
7092025-10-15T16:13:03.135Z
7102025-10-15T16:13:03.135Zreal 16:17.232189242
7112025-10-15T16:13:03.135Zuser 40:39.816526958
7122025-10-15T16:13:03.135Zsys 33.159850915
7132025-10-15T16:13:03.135Ztrap 1.422047531
7142025-10-15T16:13:03.135Ztflt 0.056442818
7152025-10-15T16:13:03.135Zdflt 0.077888771
7162025-10-15T16:13:03.135Zkflt 0.001206389
7172025-10-15T16:13:03.135Zlock 23:51:56.527847142
7182025-10-15T16:13:03.135Zslp 1:13:41.911279695
7192025-10-15T16:13:03.135Zlat 4:56.282693961
7202025-10-15T16:13:03.135Zstop 0.055479487
7212025-10-15T16:13:03.135Z+ echo 'test-up-2region-encrypted ends'
7222025-10-15T16:13:03.135Ztest-up-2region-encrypted ends
7232025-10-15T16:13:08.137Zprocess exited: duration 987243 ms, exit code 0
7242025-10-15T16:13:08.137Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
7252025-10-15T16:14:08.175Zexec warning: : stdout descriptor may be held open by a background process; giving up!
7262025-10-15T16:14:08.175Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
7272025-10-15T16:14:08.188Zfound 17 output files
7282025-10-15T16:14:08.188Zuploading: /tmp/test_up-build/dsc-out.txt (55794 bytes)
7292025-10-15T16:14:09.202Zuploaded: /tmp/test_up-build/dsc-out.txt
7302025-10-15T16:14:09.205Zuploading: /tmp/test_up-build/test_up_out.txt (20987682 bytes)
7312025-10-15T16:14:10.408Zuploaded: /tmp/test_up-build/test_up_out.txt
7322025-10-15T16:14:10.411Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (550850 bytes)
7332025-10-15T16:14:11.430Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
7342025-10-15T16:14:11.432Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (548229 bytes)
7352025-10-15T16:14:12.451Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
7362025-10-15T16:14:12.454Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (979656 bytes)
7372025-10-15T16:14:13.474Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
7382025-10-15T16:14:13.477Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (543003 bytes)
7392025-10-15T16:14:14.496Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
7402025-10-15T16:14:14.499Zuploading: /tmp/test_up-build/dsc/downstairs-8850.txt (1341026 bytes)
7412025-10-15T16:14:15.525Zuploaded: /tmp/test_up-build/dsc/downstairs-8850.txt
7422025-10-15T16:14:15.528Zuploading: /tmp/test_up-build/dsc/downstairs-8860.txt (41629 bytes)
7432025-10-15T16:14:16.539Zuploaded: /tmp/test_up-build/dsc/downstairs-8860.txt
7442025-10-15T16:14:16.542Zuploading: /tmp/test_up-build/dsc/downstairs-8870.txt (3706473 bytes)
7452025-10-15T16:14:17.587Zuploaded: /tmp/test_up-build/dsc/downstairs-8870.txt
7462025-10-15T16:14:17.590Zuploading: /tmp/debug/df.txt (1270 bytes)
7472025-10-15T16:14:18.600Zuploaded: /tmp/debug/df.txt
7482025-10-15T16:14:18.603Zuploading: /tmp/debug/dtrace.txt (1425275 bytes)
7492025-10-15T16:14:19.633Zuploaded: /tmp/debug/dtrace.txt
7502025-10-15T16:14:19.633Zuploading: /tmp/debug/iostat.txt (313148 bytes)
7512025-10-15T16:14:19.647Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 313148 -> 316448
7522025-10-15T16:14:20.654Zuploaded: /tmp/debug/iostat.txt
7532025-10-15T16:14:20.657Zuploading: /tmp/debug/mpstat.txt (760374 bytes)
7542025-10-15T16:14:20.671Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 760374 -> 769122
7552025-10-15T16:14:21.681Zuploaded: /tmp/debug/mpstat.txt
7562025-10-15T16:14:21.684Zuploading: /tmp/debug/paging.txt (136535 bytes)
7572025-10-15T16:14:21.691Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 136535 -> 138188
7582025-10-15T16:14:22.701Zuploaded: /tmp/debug/paging.txt
7592025-10-15T16:14:22.701Zuploading: /tmp/debug/prstat.txt (1501991 bytes)
7602025-10-15T16:14:22.723Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 1501991 -> 1521365
7612025-10-15T16:14:23.730Zuploaded: /tmp/debug/prstat.txt
7622025-10-15T16:14:23.733Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
7632025-10-15T16:14:24.740Zuploaded: /tmp/debug/psrinfo.txt
7642025-10-15T16:14:24.743Zuploading: /tmp/debug/upstairs-info.txt (235620 bytes)
7652025-10-15T16:14:25.755Zuploaded: /tmp/debug/upstairs-info.txt