01KBHG84DC3CN3DW5CAD20PRG9: test-up-2region-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01KBHG8ERX6Q22PV1QRWCWGGGZ

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12025-12-03T07:20:35.158Zjob dependencies complete; ready to run (waiting for 18 m 58 s)
22025-12-03T07:21:33.334Zjob assigned to worker 01KBHHBBBJXNE4VXXX13943CNR [factory aws, i-0d269541f75368281] (queued for 58 s)
32025-12-03T07:21:42.098Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42025-12-03T07:21:43.549Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52025-12-03T07:21:43.549Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62025-12-03T07:21:44.942Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72025-12-03T07:21:44.943Zdownloading input: /input/build/work/bins/crutest.gz
82025-12-03T07:21:46.406Zdownloaded input: /input/build/work/bins/crutest.gz
92025-12-03T07:21:46.406Zdownloading input: /input/build/work/bins/dsc.gz
102025-12-03T07:21:47.135Zdownloaded input: /input/build/work/bins/dsc.gz
112025-12-03T07:21:47.139Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122025-12-03T07:21:47.159Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132025-12-03T07:21:47.159Zdownloading input: /input/build/work/scripts/test_ds.sh
142025-12-03T07:21:47.181Zdownloaded input: /input/build/work/scripts/test_ds.sh
152025-12-03T07:21:47.181Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162025-12-03T07:21:47.205Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172025-12-03T07:21:47.205Zdownloading input: /input/build/work/scripts/test_repair.sh
182025-12-03T07:21:47.225Zdownloaded input: /input/build/work/scripts/test_repair.sh
192025-12-03T07:21:47.225Zdownloading input: /input/build/work/scripts/test_replay.sh
202025-12-03T07:21:47.249Zdownloaded input: /input/build/work/scripts/test_replay.sh
212025-12-03T07:21:47.249Zdownloading input: /input/build/work/scripts/test_up.sh
222025-12-03T07:21:47.296Zdownloaded input: /input/build/work/scripts/test_up.sh
232025-12-03T07:21:47.296Zdownloading input: /input/build/work/scripts/upstairs_info.d
242025-12-03T07:21:47.323Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252025-12-03T07:21:47.327Zdownloading input: /input/build/tmp/cargo-test-out.log
262025-12-03T07:21:47.885Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272025-12-03T07:21:47.885Zstarting task 0: "setup"
282025-12-03T07:21:47.891Z++ uname -s
292025-12-03T07:21:47.894Z+ kern=SunOS
302025-12-03T07:21:47.895Z+ build_user=build
312025-12-03T07:21:47.895Z+ build_uid=12345
322025-12-03T07:21:47.895Z+ work_dir=/work
332025-12-03T07:21:47.895Z+ input_dir=/input
342025-12-03T07:21:47.895Z+ [[ 0 == 12345 ]]
352025-12-03T07:21:47.895Z+ case "$kern" in
362025-12-03T07:21:47.895Z+ groupadd -g 12345 build
372025-12-03T07:21:47.895Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382025-12-03T07:21:49.903Z+ zfs create -o mountpoint=/work rpool/work
392025-12-03T07:21:50.267Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402025-12-03T07:21:50.271Z+ home_fs=zfs
412025-12-03T07:21:50.271Z+ [[ zfs == autofs ]]
422025-12-03T07:21:50.271Z+ mkdir -p /home/build
432025-12-03T07:21:50.276Z+ chown build:build /home/build /work
442025-12-03T07:21:51.273Z+ chmod 0700 /home/build /work
452025-12-03T07:21:51.276Zprocess exited: duration 3394 ms, exit code 0
 
462025-12-03T07:21:51.281Zstarting task 1: "authentication"
472025-12-03T07:21:51.303Zprocess exited: duration 20 ms, exit code 0
 
482025-12-03T07:21:51.311Zstarting task 2: "build"
492025-12-03T07:21:51.314Z+ banner cores
502025-12-03T07:21:51.316Z
512025-12-03T07:21:51.317Z #### #### ##### ###### ####
522025-12-03T07:21:51.317Z # # # # # # # #
532025-12-03T07:21:51.317Z # # # # # ##### ####
542025-12-03T07:21:51.317Z # # # ##### # #
552025-12-03T07:21:51.317Z # # # # # # # # #
562025-12-03T07:21:51.317Z #### #### # # ###### ####
572025-12-03T07:21:51.317Z
582025-12-03T07:21:51.319Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
592025-12-03T07:21:51.326Z+ banner unpack
602025-12-03T07:21:51.329Z
612025-12-03T07:21:51.329Z # # # # ##### ## #### # #
622025-12-03T07:21:51.329Z # # ## # # # # # # # # #
632025-12-03T07:21:51.329Z # # # # # # # # # # ####
642025-12-03T07:21:51.329Z # # # # # ##### ###### # # #
652025-12-03T07:21:51.329Z # # # ## # # # # # # #
662025-12-03T07:21:51.329Z #### # # # # # #### # #
672025-12-03T07:21:51.329Z
682025-12-03T07:21:51.329Z+ mkdir -p /var/tmp/bins
692025-12-03T07:21:51.332Z+ for t in "$input/bins/"*.gz
702025-12-03T07:21:51.332Z++ basename /input/build/work/bins/crucible-downstairs.gz
712025-12-03T07:21:51.335Z+ b=crucible-downstairs.gz
722025-12-03T07:21:51.335Z+ b=crucible-downstairs
732025-12-03T07:21:51.335Z+ gunzip
742025-12-03T07:21:54.343Z+ chmod +x /var/tmp/bins/crucible-downstairs
752025-12-03T07:21:54.346Z+ for t in "$input/bins/"*.gz
762025-12-03T07:21:54.348Z++ basename /input/build/work/bins/crucible-hammer.gz
772025-12-03T07:21:54.348Z+ b=crucible-hammer.gz
782025-12-03T07:21:54.348Z+ b=crucible-hammer
792025-12-03T07:21:54.348Z+ gunzip
802025-12-03T07:21:57.193Z+ chmod +x /var/tmp/bins/crucible-hammer
812025-12-03T07:21:57.196Z+ for t in "$input/bins/"*.gz
822025-12-03T07:21:57.199Z++ basename /input/build/work/bins/crutest.gz
832025-12-03T07:21:57.199Z+ b=crutest.gz
842025-12-03T07:21:57.199Z+ b=crutest
852025-12-03T07:21:57.199Z+ gunzip
862025-12-03T07:22:00.203Z+ chmod +x /var/tmp/bins/crutest
872025-12-03T07:22:00.206Z+ for t in "$input/bins/"*.gz
882025-12-03T07:22:00.208Z++ basename /input/build/work/bins/dsc.gz
892025-12-03T07:22:00.208Z+ b=dsc.gz
902025-12-03T07:22:00.208Z+ b=dsc
912025-12-03T07:22:00.208Z+ gunzip
922025-12-03T07:22:01.653Z+ chmod +x /var/tmp/bins/dsc
932025-12-03T07:22:01.656Z+ export BINDIR=/var/tmp/bins
942025-12-03T07:22:01.656Z+ BINDIR=/var/tmp/bins
952025-12-03T07:22:01.656Z+ jobpid=1155
962025-12-03T07:22:01.659Z+ echo 'Setup debug logging'
972025-12-03T07:22:01.659ZSetup debug logging
982025-12-03T07:22:01.659Z+ mkdir /tmp/debug
992025-12-03T07:22:01.659Z+ sleep 3600
1002025-12-03T07:22:01.659Z+ psrinfo -v
1012025-12-03T07:22:01.663Z+ df -h
1022025-12-03T07:22:01.671Z+ prstat -d d -mLc 1
1032025-12-03T07:22:01.674Z+ iostat -T d -xn 1
1042025-12-03T07:22:01.674Z+ mpstat -T d 1
1052025-12-03T07:22:01.674Z+ vmstat -T d -p 1
1062025-12-03T07:22:01.677Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1072025-12-03T07:22:01.677Z+ banner test_up_2r_encrypted
1082025-12-03T07:22:01.677Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1092025-12-03T07:22:01.680Z #####
1102025-12-03T07:22:01.680Z ##### ###### #### ##### # # ##### # # #####
1112025-12-03T07:22:01.680Z # # # # # # # # # # #
1122025-12-03T07:22:01.680Z # ##### #### # # # # # ##### # #
1132025-12-03T07:22:01.680Z # # # # # # ##### # #####
1142025-12-03T07:22:01.680Z # # # # # # # # # # #
1152025-12-03T07:22:01.680Z # ###### #### # ####### #### # ####### ####### # #
1162025-12-03T07:22:01.680Z
1172025-12-03T07:22:01.680Z+ ptime -m bash /input/build/work/scripts/test_up.sh -r 2 -N encrypted
1182025-12-03T07:22:01.690Z/input/build/work
1192025-12-03T07:22:01.703ZUsing 2 region sets
1202025-12-03T07:22:01.703ZTurn off color for downstairs dump
1212025-12-03T07:22:01.722ZUpstairs using key: oxdLr7e+aGl1e+qmM8QMEL05lZG32wnrTgNELMo/aMg=
1222025-12-03T07:22:01.725Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1232025-12-03T07:22:01.725ZCreating 6 downstairs regions
1242025-12-03T07:22:02.102ZStarting 6 downstairs
1252025-12-03T07:22:02.105Zdsc started at PID: 1203
1262025-12-03T07:22:07.463ZDisable automatic restart on all downstairs
1272025-12-03T07:22:07.520Z
1282025-12-03T07:22:07.520ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1292025-12-03T07:22:07.523ZRunning test: span
1302025-12-03T07:22:11.725ZCompleted test: span
1312025-12-03T07:22:16.733ZRunning test: big
1322025-12-03T07:22:21.336ZCompleted test: big
1332025-12-03T07:22:26.344ZRunning test: dep
1342025-12-03T07:23:00.076ZCompleted test: dep
1352025-12-03T07:23:05.085ZRunning test: balloon
1362025-12-03T07:27:22.447ZCompleted test: balloon
1372025-12-03T07:27:27.456ZRunning test: deactivate
1382025-12-03T07:27:32.143ZCompleted test: deactivate
1392025-12-03T07:27:37.151ZRunning hammer
1402025-12-03T07:27:50.103ZRun repair tests
1412025-12-03T07:27:50.107Z/var/tmp/bins/crutest fill -g 31 -q --key oxdLr7e+aGl1e+qmM8QMEL05lZG32wnrTgNELMo/aMg= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1422025-12-03T07:27:50.168Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.006976821Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"task":"crutest"}
1432025-12-03T07:27:50.171Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.007691272Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"task":"crutest"}
1442025-12-03T07:27:50.171Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.009550155Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"task":"crutest"}
1452025-12-03T07:27:50.171Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.009871122Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246}
1462025-12-03T07:27:50.171Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2882d8b71cde865f6d2a2d0554debbccaa9eae34\",\n git_commit_timestamp: \"2025-12-03T07:01:06.000000000Z\",\n git_branch: \"renovate/actions-checkout-digest\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.009931041Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246}
1472025-12-03T07:27:50.171Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.009958977Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246}
1482025-12-03T07:27:50.171Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.009996741Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246}
1492025-12-03T07:27:50.174Z{"msg":"Crucible bc816c6c-54df-4a2a-a205-8fc9f2483d2e has session id: 05e06e93-2eed-48b5-9b3b-286745486339","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.010585353Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1502025-12-03T07:27:50.174Z{"msg":"Upstairs opts: Upstairs UUID: bc816c6c-54df-4a2a-a205-8fc9f2483d2e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.01066491Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1512025-12-03T07:27:50.174Z{"msg":"Crucible stats registered with UUID: bc816c6c-54df-4a2a-a205-8fc9f2483d2e","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.010710492Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1522025-12-03T07:27:50.174Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-12-03T07:27:49.010739118Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1532025-12-03T07:27:50.174Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.010944654Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1542025-12-03T07:27:50.174Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.011039128Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1552025-12-03T07:27:50.174Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.011072143Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1562025-12-03T07:27:50.174Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.012954951Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"task":"crutest"}
1572025-12-03T07:27:50.174Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.013063373Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246}
1582025-12-03T07:27:50.174Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2882d8b71cde865f6d2a2d0554debbccaa9eae34\",\n git_commit_timestamp: \"2025-12-03T07:01:06.000000000Z\",\n git_branch: \"renovate/actions-checkout-digest\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.013118814Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246}
1592025-12-03T07:27:50.174Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.01314243Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246}
1602025-12-03T07:27:50.174Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.013165586Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246}
1612025-12-03T07:27:50.178Z{"msg":"Crucible bc816c6c-54df-4a2a-a205-8fc9f2483d2e has session id: ffbd9029-5a4e-45ea-97c9-8b68bb5c96df","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.013612522Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
1622025-12-03T07:27:50.178Z{"msg":"Upstairs opts: Upstairs UUID: bc816c6c-54df-4a2a-a205-8fc9f2483d2e, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.013681571Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
1632025-12-03T07:27:50.178Z{"msg":"Crucible stats registered with UUID: bc816c6c-54df-4a2a-a205-8fc9f2483d2e","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.013724054Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
1642025-12-03T07:27:50.178Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-12-03T07:27:49.013751979Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
1652025-12-03T07:27:50.178Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.013780744Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
1662025-12-03T07:27:50.178Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.013799801Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
1672025-12-03T07:27:50.178Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.013824977Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
1682025-12-03T07:27:50.178Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.014481378Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"task":"crutest"}
1692025-12-03T07:27:50.178Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.014503605Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246}
1702025-12-03T07:27:50.178Z{"msg":"bc816c6c-54df-4a2a-a205-8fc9f2483d2e active request set","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.014608867Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1712025-12-03T07:27:50.178Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.014647421Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"client":"0","":"downstairs","session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1722025-12-03T07:27:50.178Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.014702062Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"client":"1","":"downstairs","session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1732025-12-03T07:27:50.178Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.014756873Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"client":"2","":"downstairs","session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1742025-12-03T07:27:50.178Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.014795876Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1752025-12-03T07:27:50.178Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.014842239Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1762025-12-03T07:27:50.178Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.014874633Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1772025-12-03T07:27:50.178Z{"msg":"ds_connection connected from Ok(127.0.0.1:36072)","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.014904398Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1782025-12-03T07:27:50.178Z{"msg":"ds_connection connected from Ok(127.0.0.1:37044)","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.014946062Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1792025-12-03T07:27:50.178Z{"msg":"ds_connection connected from Ok(127.0.0.1:62324)","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.014985955Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1802025-12-03T07:27:50.178Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.016257454Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"client":"0","":"downstairs","session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1812025-12-03T07:27:50.178Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.016367416Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"client":"1","":"downstairs","session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1822025-12-03T07:27:50.178Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.016516902Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"client":"2","":"downstairs","session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1832025-12-03T07:27:50.178Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.016724197Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1842025-12-03T07:27:50.178Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.016780698Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1852025-12-03T07:27:50.183Z{"msg":"[0]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.016883201Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"downstairs","":"downstairs","session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1862025-12-03T07:27:50.183Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.016917325Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"downstairs","":"downstairs","session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1872025-12-03T07:27:50.183Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.0169505Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"downstairs","":"downstairs","session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1882025-12-03T07:27:50.183Z{"msg":"[1]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.016991523Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"downstairs","":"downstairs","session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1892025-12-03T07:27:50.183Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.017049843Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"downstairs","":"downstairs","session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1902025-12-03T07:27:50.183Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.017086517Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"downstairs","":"downstairs","session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1912025-12-03T07:27:50.183Z{"msg":"[2]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.017114453Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"downstairs","":"downstairs","session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1922025-12-03T07:27:50.183Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.017144608Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"downstairs","":"downstairs","session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1932025-12-03T07:27:50.183Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.017178092Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"downstairs","":"downstairs","session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1942025-12-03T07:27:50.183Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.017212706Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"downstairs","":"downstairs","session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1952025-12-03T07:27:50.183Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.017234763Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"downstairs","":"downstairs","session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1962025-12-03T07:27:50.183Z{"msg":"Next flush: 3999","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.017267067Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"downstairs","":"downstairs","session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1972025-12-03T07:27:50.183Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.017286334Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"downstairs","":"downstairs","session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1982025-12-03T07:27:50.183Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.0173138Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
1992025-12-03T07:27:50.183Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.017337576Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"client":"0","":"downstairs","session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
2002025-12-03T07:27:50.183Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.01736871Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"client":"1","":"downstairs","session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
2012025-12-03T07:27:50.183Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.017397336Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"client":"2","":"downstairs","session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
2022025-12-03T07:27:50.183Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.017485941Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
2032025-12-03T07:27:50.183Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.017644595Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
2042025-12-03T07:27:50.183Z{"msg":"bc816c6c-54df-4a2a-a205-8fc9f2483d2e is now active with session: 05e06e93-2eed-48b5-9b3b-286745486339","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.017692457Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
2052025-12-03T07:27:50.183Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.017720632Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"session_id":"05e06e93-2eed-48b5-9b3b-286745486339"}
2062025-12-03T07:27:50.183Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.017738579Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246}
2072025-12-03T07:27:50.183Z{"msg":"Activated sub_volume bc816c6c-54df-4a2a-a205-8fc9f2483d2e","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.017755506Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246}
2082025-12-03T07:27:50.183Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.017786212Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246}
2092025-12-03T07:27:50.183Z{"msg":"bc816c6c-54df-4a2a-a205-8fc9f2483d2e active request set","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.017817726Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2102025-12-03T07:27:50.183Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.017851291Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"client":"0","":"downstairs","session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2112025-12-03T07:27:50.183Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.017892134Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"client":"1","":"downstairs","session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2122025-12-03T07:27:50.183Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.017943135Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"client":"2","":"downstairs","session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2132025-12-03T07:27:50.183Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.017983949Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2142025-12-03T07:27:50.183Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.018020873Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2152025-12-03T07:27:50.183Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.018063246Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2162025-12-03T07:27:50.184Z{"msg":"ds_connection connected from Ok(127.0.0.1:50214)","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.01809704Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2172025-12-03T07:27:50.184Z{"msg":"ds_connection connected from Ok(127.0.0.1:32864)","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.018137523Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2182025-12-03T07:27:50.184Z{"msg":"ds_connection connected from Ok(127.0.0.1:52347)","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.018170978Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2192025-12-03T07:27:50.184Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.018720937Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"client":"1","":"downstairs","session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2202025-12-03T07:27:50.184Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.018781337Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"client":"2","":"downstairs","session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2212025-12-03T07:27:50.184Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.018837927Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"client":"0","":"downstairs","session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2222025-12-03T07:27:50.184Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.019099584Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2232025-12-03T07:27:50.184Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.019139137Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2242025-12-03T07:27:50.184Z{"msg":"[0]R flush_numbers: [5316, 5319, 5319, 5319, 5319]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.019157254Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"downstairs","":"downstairs","session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2252025-12-03T07:27:50.184Z{"msg":"[0]R generation: [13, 14, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.0191821Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"downstairs","":"downstairs","session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2262025-12-03T07:27:50.184Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.019199727Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"downstairs","":"downstairs","session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2272025-12-03T07:27:50.184Z{"msg":"[1]R flush_numbers: [5316, 5319, 5319, 5319, 5319]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.019229513Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"downstairs","":"downstairs","session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2282025-12-03T07:27:50.184Z{"msg":"[1]R generation: [13, 14, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.01924723Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"downstairs","":"downstairs","session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2292025-12-03T07:27:50.184Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.019271166Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"downstairs","":"downstairs","session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2302025-12-03T07:27:50.184Z{"msg":"[2]R flush_numbers: [5316, 5319, 5319, 5319, 5319]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.019289123Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"downstairs","":"downstairs","session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2312025-12-03T07:27:50.184Z{"msg":"[2]R generation: [13, 14, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.019312789Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"downstairs","":"downstairs","session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2322025-12-03T07:27:50.184Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.019330866Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"downstairs","":"downstairs","session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2332025-12-03T07:27:50.184Z{"msg":"Max found gen is 15","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.019353472Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"downstairs","":"downstairs","session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2342025-12-03T07:27:50.184Z{"msg":"Generation requested: 31 >= found:15","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.019371069Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"downstairs","":"downstairs","session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2352025-12-03T07:27:50.184Z{"msg":"Next flush: 5320","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.019386907Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"downstairs","":"downstairs","session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2362025-12-03T07:27:50.184Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.019410373Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"":"downstairs","":"downstairs","session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}Wait for a query_work_queue command to finish before sending IO
2372025-12-03T07:27:50.184Z
2382025-12-03T07:27:50.184Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.019480371Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2392025-12-03T07:27:50.184Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.019507316Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"client":"0","":"downstairs","session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2402025-12-03T07:27:50.184Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.019525593Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"client":"1","":"downstairs","session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2412025-12-03T07:27:50.184Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.01954949Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"client":"2","":"downstairs","session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2422025-12-03T07:27:50.184Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.019567517Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2432025-12-03T07:27:50.184Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.019590353Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2442025-12-03T07:27:50.184Z{"msg":"bc816c6c-54df-4a2a-a205-8fc9f2483d2e is now active with session: ffbd9029-5a4e-45ea-97c9-8b68bb5c96df","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.01960868Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2452025-12-03T07:27:50.184Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.019630186Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246,"session_id":"ffbd9029-5a4e-45ea-97c9-8b68bb5c96df"}
2462025-12-03T07:27:50.184Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.019647373Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246}
2472025-12-03T07:27:50.184Z{"msg":"Activated sub_volume bc816c6c-54df-4a2a-a205-8fc9f2483d2e","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:49.01966807Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246}
2482025-12-03T07:27:50.184ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
2492025-12-03T07:27:50.184ZFill test
2502025-12-03T07:27:50.224ZRead and Verify all blocks (0..100 range:false)
2512025-12-03T07:27:50.304ZWrote out file "/var/tmp/test_up-build/verify_file"
2522025-12-03T07:27:50.304ZCLIENT: Tests done. All submitted work has been ACK'd
2532025-12-03T07:27:50.308Z----------------------------------------------------------------
2542025-12-03T07:27:50.308Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2552025-12-03T07:27:50.308ZStates: Active Active Active
2562025-12-03T07:27:50.308ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2572025-12-03T07:27:50.308Z 1002 Acked Read 50 Done Done Done false
2582025-12-03T07:27:50.308Z STATES DS:0 DS:1 DS:2 TOTAL
2592025-12-03T07:27:50.308Z Sent 0 0 0 0
2602025-12-03T07:27:50.308Z Done 1 1 1 3
2612025-12-03T07:27:50.308Z Skipped 0 0 0 0
2622025-12-03T07:27:50.308Z Error 0 0 0 0
2632025-12-03T07:27:50.308ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2642025-12-03T07:27:50.308ZDownstairs last five completed: 1001 1000
2652025-12-03T07:27:50.308ZUpstairs last five completed: 1002 1001 1000
2662025-12-03T07:27:50.308Z----------------------------------------------------------------
2672025-12-03T07:27:50.308Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2682025-12-03T07:27:50.308ZStates: Active Active Active
2692025-12-03T07:27:50.308ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2702025-12-03T07:27:50.308Z 1002 Acked Read 50 Done Done Done false
2712025-12-03T07:27:50.308Z STATES DS:0 DS:1 DS:2 TOTAL
2722025-12-03T07:27:50.308Z Sent 0 0 0 0
2732025-12-03T07:27:50.308Z Done 1 1 1 3
2742025-12-03T07:27:50.309Z Skipped 0 0 0 0
2752025-12-03T07:27:50.309Z Error 0 0 0 0
2762025-12-03T07:27:50.309ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2772025-12-03T07:27:50.309ZDownstairs last five completed: 1001 1000
2782025-12-03T07:27:50.309ZUpstairs last five completed: 1002 1001 1000
2792025-12-03T07:27:50.309ZCLIENT: Up:0 ds:2 act:6
2802025-12-03T07:27:54.307Z----------------------------------------------------------------
2812025-12-03T07:27:54.307Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2822025-12-03T07:27:54.307ZDownstairs last five completed: 1003 1002 1001 1000
2832025-12-03T07:27:54.307ZUpstairs last five completed: 1003 1002 1001 1000
2842025-12-03T07:27:54.310Z----------------------------------------------------------------
2852025-12-03T07:27:54.310Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2862025-12-03T07:27:54.310ZDownstairs last five completed: 1003 1002 1001 1000
2872025-12-03T07:27:54.310ZUpstairs last five completed: 1003 1002 1001 1000
2882025-12-03T07:27:54.310ZCLIENT: Up:0 ds:0 act:6
2892025-12-03T07:27:54.310ZCLIENT: All crucible jobs finished, exiting program
2902025-12-03T07:27:54.310Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-12-03T07:27:53.145791615Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1246}
2912025-12-03T07:27:54.319ZRepair setup passed
2922025-12-03T07:27:54.323ZCopy the region for /var/tmp/test_up-build/8830
2932025-12-03T07:27:54.326Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2942025-12-03T07:27:54.329Z/var/tmp/bins/crutest fill -g 32 -q --key oxdLr7e+aGl1e+qmM8QMEL05lZG32wnrTgNELMo/aMg= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2952025-12-03T07:27:54.388Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.225976904Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"task":"crutest"}
2962025-12-03T07:27:54.391Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.22660316Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"task":"crutest"}
2972025-12-03T07:27:54.391Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.228411191Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"task":"crutest"}
2982025-12-03T07:27:54.391Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.228739517Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252}
2992025-12-03T07:27:54.391Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2882d8b71cde865f6d2a2d0554debbccaa9eae34\",\n git_commit_timestamp: \"2025-12-03T07:01:06.000000000Z\",\n git_branch: \"renovate/actions-checkout-digest\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.228791638Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252}
3002025-12-03T07:27:54.391Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.228815154Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252}
3012025-12-03T07:27:54.391Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.22883883Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252}
3022025-12-03T07:27:54.395Z{"msg":"Crucible 4dced55d-5aed-4126-9c39-1c5c98d9ec74 has session id: a07b8aa8-4013-4b10-8d70-262e6458ec81","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.229455408Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3032025-12-03T07:27:54.395Z{"msg":"Upstairs opts: Upstairs UUID: 4dced55d-5aed-4126-9c39-1c5c98d9ec74, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.229526696Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3042025-12-03T07:27:54.395Z{"msg":"Crucible stats registered with UUID: 4dced55d-5aed-4126-9c39-1c5c98d9ec74","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.229582837Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3052025-12-03T07:27:54.395Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-12-03T07:27:53.229621521Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3062025-12-03T07:27:54.395Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.229814859Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3072025-12-03T07:27:54.395Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.22986811Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3082025-12-03T07:27:54.395Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.229888886Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3092025-12-03T07:27:54.395Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.231819057Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"task":"crutest"}
3102025-12-03T07:27:54.395Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.231912211Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252}
3112025-12-03T07:27:54.395Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2882d8b71cde865f6d2a2d0554debbccaa9eae34\",\n git_commit_timestamp: \"2025-12-03T07:01:06.000000000Z\",\n git_branch: \"renovate/actions-checkout-digest\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.231962273Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252}
3122025-12-03T07:27:54.395Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.23198525Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252}
3132025-12-03T07:27:54.395Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.232021393Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252}
3142025-12-03T07:27:54.395Z{"msg":"Crucible 4dced55d-5aed-4126-9c39-1c5c98d9ec74 has session id: df8d35d0-5270-4212-b033-d00943b28a4b","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.232288549Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3152025-12-03T07:27:54.395Z{"msg":"Upstairs opts: Upstairs UUID: 4dced55d-5aed-4126-9c39-1c5c98d9ec74, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.232350929Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3162025-12-03T07:27:54.395Z{"msg":"Crucible stats registered with UUID: 4dced55d-5aed-4126-9c39-1c5c98d9ec74","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.232385673Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3172025-12-03T07:27:54.395Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-12-03T07:27:53.232412609Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3182025-12-03T07:27:54.395Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.232482057Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3192025-12-03T07:27:54.395Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.232545747Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3202025-12-03T07:27:54.395Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.232589279Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3212025-12-03T07:27:54.399Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.233341995Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"task":"crutest"}
3222025-12-03T07:27:54.399Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.233377469Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252}
3232025-12-03T07:27:54.399Z{"msg":"4dced55d-5aed-4126-9c39-1c5c98d9ec74 active request set","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.233442478Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3242025-12-03T07:27:54.399Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.23348981Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"client":"0","":"downstairs","session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3252025-12-03T07:27:54.399Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.233532034Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"client":"1","":"downstairs","session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3262025-12-03T07:27:54.399Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.233559089Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"client":"2","":"downstairs","session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3272025-12-03T07:27:54.399Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.233606721Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3282025-12-03T07:27:54.399Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.233631557Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3292025-12-03T07:27:54.399Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.233662822Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3302025-12-03T07:27:54.399Z{"msg":"ds_connection connected from Ok(127.0.0.1:62325)","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.233775943Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3312025-12-03T07:27:54.399Z{"msg":"ds_connection connected from Ok(127.0.0.1:52890)","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.233827825Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3322025-12-03T07:27:54.399Z{"msg":"ds_connection connected from Ok(127.0.0.1:50523)","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.233865608Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3332025-12-03T07:27:54.399Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.235167583Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"client":"1","":"downstairs","session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3342025-12-03T07:27:54.399Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.235225203Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"client":"2","":"downstairs","session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3352025-12-03T07:27:54.399Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.235274065Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"client":"0","":"downstairs","session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3362025-12-03T07:27:54.399Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.235562447Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3372025-12-03T07:27:54.399Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.235598732Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3382025-12-03T07:27:54.399Z{"msg":"[0]R flush_numbers: [3999, 3999, 3999, 3999, 3999]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.235617318Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"downstairs","":"downstairs","session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3392025-12-03T07:27:54.399Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.235635605Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"downstairs","":"downstairs","session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3402025-12-03T07:27:54.399Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.235653872Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"downstairs","":"downstairs","session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3412025-12-03T07:27:54.399Z{"msg":"[1]R flush_numbers: [3999, 3999, 3999, 3999, 3999]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.235679208Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"downstairs","":"downstairs","session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3422025-12-03T07:27:54.399Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.235696895Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"downstairs","":"downstairs","session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3432025-12-03T07:27:54.399Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.235720621Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"downstairs","":"downstairs","session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3442025-12-03T07:27:54.399Z{"msg":"[2]R flush_numbers: [3999, 3999, 3999, 3999, 3999]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.235739648Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"downstairs","":"downstairs","session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3452025-12-03T07:27:54.399Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.235763274Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"downstairs","":"downstairs","session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3462025-12-03T07:27:54.399Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.235780791Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"downstairs","":"downstairs","session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3472025-12-03T07:27:54.399Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.235803418Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"downstairs","":"downstairs","session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3482025-12-03T07:27:54.399Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.235820765Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"downstairs","":"downstairs","session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3492025-12-03T07:27:54.399Z{"msg":"Next flush: 4000","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.235837362Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"downstairs","":"downstairs","session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3502025-12-03T07:27:54.399Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.235853749Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"downstairs","":"downstairs","session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3512025-12-03T07:27:54.399Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.235878165Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3522025-12-03T07:27:54.400Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.235893843Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"client":"0","":"downstairs","session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3532025-12-03T07:27:54.400Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.235917069Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"client":"1","":"downstairs","session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3542025-12-03T07:27:54.400Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.235933986Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"client":"2","":"downstairs","session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3552025-12-03T07:27:54.400Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.23596814Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3562025-12-03T07:27:54.400Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.235986137Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3572025-12-03T07:27:54.400Z{"msg":"4dced55d-5aed-4126-9c39-1c5c98d9ec74 is now active with session: a07b8aa8-4013-4b10-8d70-262e6458ec81","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.23602965Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3582025-12-03T07:27:54.400Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.236054606Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"session_id":"a07b8aa8-4013-4b10-8d70-262e6458ec81"}
3592025-12-03T07:27:54.400Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.236079222Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252}
3602025-12-03T07:27:54.400Z{"msg":"Activated sub_volume 4dced55d-5aed-4126-9c39-1c5c98d9ec74","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.236115626Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252}
3612025-12-03T07:27:54.400Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.236162898Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252}
3622025-12-03T07:27:54.400Z{"msg":"4dced55d-5aed-4126-9c39-1c5c98d9ec74 active request set","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.236284738Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3632025-12-03T07:27:54.400Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.236348587Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"client":"0","":"downstairs","session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3642025-12-03T07:27:54.400Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.236381152Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"client":"1","":"downstairs","session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3652025-12-03T07:27:54.400Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.236415436Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"client":"2","":"downstairs","session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3662025-12-03T07:27:54.400Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.236445251Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3672025-12-03T07:27:54.400Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.236485565Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3682025-12-03T07:27:54.400Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.236520849Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3692025-12-03T07:27:54.400Z{"msg":"ds_connection connected from Ok(127.0.0.1:48997)","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.236554373Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3702025-12-03T07:27:54.400Z{"msg":"ds_connection connected from Ok(127.0.0.1:40010)","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.236584248Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3712025-12-03T07:27:54.403Z{"msg":"ds_connection connected from Ok(127.0.0.1:49338)","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.236636Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3722025-12-03T07:27:54.403Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.237165392Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"client":"2","":"downstairs","session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3732025-12-03T07:27:54.404Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.237250988Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"client":"1","":"downstairs","session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3742025-12-03T07:27:54.404Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.237407592Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"client":"0","":"downstairs","session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3752025-12-03T07:27:54.404Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.23748148Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3762025-12-03T07:27:54.404Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.237654211Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3772025-12-03T07:27:54.404Z{"msg":"[0]R flush_numbers: [5320, 5320, 5320, 5320, 5320]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.237696404Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"downstairs","":"downstairs","session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3782025-12-03T07:27:54.404Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.237716471Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"downstairs","":"downstairs","session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3792025-12-03T07:27:54.404Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.237743057Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"downstairs","":"downstairs","session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3802025-12-03T07:27:54.404Z{"msg":"[1]R flush_numbers: [5320, 5320, 5320, 5320, 5320]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.237761354Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"downstairs","":"downstairs","session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3812025-12-03T07:27:54.404Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.237785319Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"downstairs","":"downstairs","session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3822025-12-03T07:27:54.404Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.237802827Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"downstairs","":"downstairs","session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3832025-12-03T07:27:54.404Z{"msg":"[2]R flush_numbers: [5320, 5320, 5320, 5320, 5320]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.237820794Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"downstairs","":"downstairs","session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3842025-12-03T07:27:54.404Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.237837861Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","Wait for a query_work_queue command to finish before sending IO
3852025-12-03T07:27:54.404Zpid":1252,"":"downstairs","":"downstairs","session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3862025-12-03T07:27:54.404Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.237866496Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"downstairs","":"downstairs","session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3872025-12-03T07:27:54.404Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.237882543Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"downstairs","":"downstairs","session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3882025-12-03T07:27:54.404Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.23790012Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"downstairs","":"downstairs","session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3892025-12-03T07:27:54.404Z{"msg":"Next flush: 5321","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.237916288Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"downstairs","":"downstairs","session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3902025-12-03T07:27:54.404Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.237939754Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"":"downstairs","":"downstairs","session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3912025-12-03T07:27:54.404Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.237957301Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3922025-12-03T07:27:54.404Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.237979487Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"client":"0","":"downstairs","session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3932025-12-03T07:27:54.404Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.238014522Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"client":"1","":"downstairs","session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3942025-12-03T07:27:54.404Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.238042417Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"client":"2","":"downstairs","session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3952025-12-03T07:27:54.404Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.238059794Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3962025-12-03T07:27:54.404Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.23808408Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3972025-12-03T07:27:54.404Z{"msg":"4dced55d-5aed-4126-9c39-1c5c98d9ec74 is now active with session: df8d35d0-5270-4212-b033-d00943b28a4b","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.238113775Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3982025-12-03T07:27:54.404Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.238152909Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252,"session_id":"df8d35d0-5270-4212-b033-d00943b28a4b"}
3992025-12-03T07:27:54.404Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.238197331Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252}
4002025-12-03T07:27:54.404Z{"msg":"Disk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
4012025-12-03T07:27:54.404ZActivated sub_volume 4dced55d-5aed-4126-9c39-1c5c98d9ec74","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:53.238239124Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252}
4022025-12-03T07:27:54.404ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4032025-12-03T07:27:54.404ZFill test
4042025-12-03T07:27:54.443ZRead and Verify all blocks (0..100 range:false)
4052025-12-03T07:27:54.522ZWrote out file "/var/tmp/test_up-build/verify_file"
4062025-12-03T07:27:54.522ZCLIENT: Tests done. All submitted work has been ACK'd
4072025-12-03T07:27:54.525Z----------------------------------------------------------------
4082025-12-03T07:27:54.525Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4092025-12-03T07:27:54.525ZStates: Active Active Active
4102025-12-03T07:27:54.525ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4112025-12-03T07:27:54.525Z 1002 Acked Read 50 Done Done Done false
4122025-12-03T07:27:54.525Z STATES DS:0 DS:1 DS:2 TOTAL
4132025-12-03T07:27:54.525Z Sent 0 0 0 0
4142025-12-03T07:27:54.525Z Done 1 1 1 3
4152025-12-03T07:27:54.526Z Skipped 0 0 0 0
4162025-12-03T07:27:54.526Z Error 0 0 0 0
4172025-12-03T07:27:54.526ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4182025-12-03T07:27:54.526ZDownstairs last five completed: 1001 1000
4192025-12-03T07:27:54.526ZUpstairs last five completed: 1002 1001 1000
4202025-12-03T07:27:54.526Z----------------------------------------------------------------
4212025-12-03T07:27:54.526Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4222025-12-03T07:27:54.526ZStates: Active Active Active
4232025-12-03T07:27:54.526ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4242025-12-03T07:27:54.526Z 1002 Acked Read 50 Done Done Done false
4252025-12-03T07:27:54.526Z STATES DS:0 DS:1 DS:2 TOTAL
4262025-12-03T07:27:54.526Z Sent 0 0 0 0
4272025-12-03T07:27:54.526Z Done 1 1 1 3
4282025-12-03T07:27:54.526Z Skipped 0 0 0 0
4292025-12-03T07:27:54.526Z Error 0 0 0 0
4302025-12-03T07:27:54.526ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4312025-12-03T07:27:54.526ZDownstairs last five completed: 1001 1000
4322025-12-03T07:27:54.526ZUpstairs last five completed: 1002 1001 1000
4332025-12-03T07:27:54.526ZCLIENT: Up:0 ds:2 act:6
4342025-12-03T07:27:58.525Z----------------------------------------------------------------
4352025-12-03T07:27:58.525Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4362025-12-03T07:27:58.525ZDownstairs last five completed: 1003 1002 1001 1000
4372025-12-03T07:27:58.525ZUpstairs last five completed: 1003 1002 1001 1000
4382025-12-03T07:27:58.528Z----------------------------------------------------------------
4392025-12-03T07:27:58.529Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4402025-12-03T07:27:58.529ZDownstairs last five completed: 1003 1002 1001 1000
4412025-12-03T07:27:58.529ZUpstairs last five completed: 1003 1002 1001 1000
4422025-12-03T07:27:58.529ZCLIENT: Up:0 ds:0 act:6
4432025-12-03T07:27:58.529ZCLIENT: All crucible jobs finished, exiting program
4442025-12-03T07:27:58.529Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-12-03T07:27:57.362633086Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1252}
4452025-12-03T07:27:58.533ZRepair part 1 passed
4462025-12-03T07:27:58.535Z
4472025-12-03T07:27:58.535ZKill the current downstairs
4482025-12-03T07:27:58.708ZDownstairs 2 stopped
4492025-12-03T07:27:58.708Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
4502025-12-03T07:27:58.711ZNow put back the original so we have a mismatch
4512025-12-03T07:27:58.711Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
4522025-12-03T07:27:58.714ZRestart downstairs with old directory
4532025-12-03T07:27:58.773Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4542025-12-03T07:27:58.792ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4552025-12-03T07:27:58.795Z 0 000-009 32 32 31 4000 4000 3999 F F F <---
4562025-12-03T07:27:58.795Z 1 010-019 32 32 31 4000 4000 3999 F F F <---
4572025-12-03T07:27:58.795Z 2 020-029 32 32 31 4000 4000 3999 F F F <---
4582025-12-03T07:27:58.795Z 3 030-039 32 32 31 4000 4000 3999 F F F <---
4592025-12-03T07:27:58.795Z 4 040-049 32 32 31 4000 4000 3999 F F F <---
4602025-12-03T07:27:58.795ZMax gen: 32, Max flush: 4000
4612025-12-03T07:27:58.795ZError: Difference in extent metadata found!
4622025-12-03T07:27:58.798Zdump test found error as expected
4632025-12-03T07:27:58.798Z
4642025-12-03T07:27:58.798Z
4652025-12-03T07:27:58.798Z/var/tmp/bins/crutest verify --range -g 33 -q --key oxdLr7e+aGl1e+qmM8QMEL05lZG32wnrTgNELMo/aMg= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
4662025-12-03T07:27:58.859Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.695845467Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"task":"crutest"}
4672025-12-03T07:27:58.862Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.696473773Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"task":"crutest"}
4682025-12-03T07:27:58.862Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.6984244Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"task":"crutest"}
4692025-12-03T07:27:58.862Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.698735338Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260}
4702025-12-03T07:27:58.862Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2882d8b71cde865f6d2a2d0554debbccaa9eae34\",\n git_commit_timestamp: \"2025-12-03T07:01:06.000000000Z\",\n git_branch: \"renovate/actions-checkout-digest\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.69878777Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260}
4712025-12-03T07:27:58.862Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.698812976Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260}
4722025-12-03T07:27:58.862Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.698836292Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260}
4732025-12-03T07:27:58.862Z{"msg":"Crucible f7bb9a08-7346-4907-8a92-a50bd752c443 has session id: b8a44954-7f83-40a9-9e28-64e2efafc4a4","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.699435283Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
4742025-12-03T07:27:58.865Z{"msg":"Upstairs opts: Upstairs UUID: f7bb9a08-7346-4907-8a92-a50bd752c443, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.699494323Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
4752025-12-03T07:27:58.865Z{"msg":"Crucible stats registered with UUID: f7bb9a08-7346-4907-8a92-a50bd752c443","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.699586538Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
4762025-12-03T07:27:58.865Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-12-03T07:27:57.699649587Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
4772025-12-03T07:27:58.865Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.699792703Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"io task","client":"0","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
4782025-12-03T07:27:58.865Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.699855373Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"io task","client":"1","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
4792025-12-03T07:27:58.865Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.699910774Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
4802025-12-03T07:27:58.865Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.701802041Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"task":"crutest"}
4812025-12-03T07:27:58.865Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.701907533Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260}
4822025-12-03T07:27:58.865Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2882d8b71cde865f6d2a2d0554debbccaa9eae34\",\n git_commit_timestamp: \"2025-12-03T07:01:06.000000000Z\",\n git_branch: \"renovate/actions-checkout-digest\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.701957045Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260}
4832025-12-03T07:27:58.866Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.701979452Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260}
4842025-12-03T07:27:58.866Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.702014256Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260}
4852025-12-03T07:27:58.866Z{"msg":"Crucible f7bb9a08-7346-4907-8a92-a50bd752c443 has session id: 4352b2ae-9828-475e-9443-bee84f3ea4fc","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.702339632Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
4862025-12-03T07:27:58.866Z{"msg":"Upstairs opts: Upstairs UUID: f7bb9a08-7346-4907-8a92-a50bd752c443, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.70241084Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
4872025-12-03T07:27:58.866Z{"msg":"Crucible stats registered with UUID: f7bb9a08-7346-4907-8a92-a50bd752c443","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.702456403Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
4882025-12-03T07:27:58.866Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-12-03T07:27:57.702484228Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
4892025-12-03T07:27:58.866Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.702516193Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"io task","client":"0","":"downstairs","session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
4902025-12-03T07:27:58.866Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.702544128Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"io task","client":"1","":"downstairs","session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
4912025-12-03T07:27:58.866Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.702586531Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
4922025-12-03T07:27:58.869Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.70313467Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"task":"crutest"}
4932025-12-03T07:27:58.869Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.703170555Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260}
4942025-12-03T07:27:58.869Z{"msg":"f7bb9a08-7346-4907-8a92-a50bd752c443 active request set","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.703239783Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
4952025-12-03T07:27:58.869Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.703278686Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"client":"0","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
4962025-12-03T07:27:58.869Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.703358053Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"client":"1","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
4972025-12-03T07:27:58.869Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.703408365Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"client":"2","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
4982025-12-03T07:27:58.869Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.70344144Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
4992025-12-03T07:27:58.869Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.703482123Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"io task","client":"0","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5002025-12-03T07:27:58.869Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.703508109Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"io task","client":"1","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5012025-12-03T07:27:58.870Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-12-03T07:27:57.703541453Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5022025-12-03T07:27:58.870Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-12-03T07:27:57.703607552Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5032025-12-03T07:27:58.870Z{"msg":"ds_connection connected from Ok(127.0.0.1:34108)","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.703647805Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"io task","client":"1","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5042025-12-03T07:27:58.870Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-12-03T07:27:57.70379896Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5052025-12-03T07:27:58.870Z{"msg":"ds_connection connected from Ok(127.0.0.1:41566)","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.703840004Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"io task","client":"0","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5062025-12-03T07:27:58.870Z{"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.703878887Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5072025-12-03T07:27:58.870Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.703916511Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5082025-12-03T07:27:58.870Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.704574772Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"client":"1","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5092025-12-03T07:27:58.870Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.704676905Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"client":"0","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5102025-12-03T07:27:58.870Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.704909936Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5112025-12-03T07:27:58.870Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:27:57.704953899Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5122025-12-03T07:28:08.870Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.705070594Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5132025-12-03T07:28:08.870Z{"msg":"ds_connection connected from Ok(127.0.0.1:57097)","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.705240226Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5142025-12-03T07:28:08.874Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.707164618Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"client":"2","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5152025-12-03T07:28:08.874Z{"msg":"[0]R flush_numbers: [4000, 4000, 4000, 4000, 4000]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.707607344Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"downstairs","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5162025-12-03T07:28:08.874Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.707640419Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"downstairs","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5172025-12-03T07:28:08.874Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.707659756Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"downstairs","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5182025-12-03T07:28:08.874Z{"msg":"[1]R flush_numbers: [4000, 4000, 4000, 4000, 4000]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.707678013Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"downstairs","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5192025-12-03T07:28:08.874Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.707702849Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"downstairs","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5202025-12-03T07:28:08.874Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.707736253Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"downstairs","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5212025-12-03T07:28:08.874Z{"msg":"[2]R flush_numbers: [3999, 3999, 3999, 3999, 3999]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.707775137Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"downstairs","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5222025-12-03T07:28:08.874Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.70781331Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"downstairs","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5232025-12-03T07:28:08.874Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.707843495Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"downstairs","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5242025-12-03T07:28:08.874Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.707886058Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"downstairs","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5252025-12-03T07:28:08.874Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.70793726Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"downstairs","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5262025-12-03T07:28:08.874Z{"msg":"Next flush: 4001","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.707984342Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"downstairs","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5272025-12-03T07:28:08.874Z{"msg":"extent 0 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3999, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.708052411Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5282025-12-03T07:28:08.874Z{"msg":"extent:0 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3999, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.708122429Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5292025-12-03T07:28:08.874Z{"msg":"found dest for source 0 for extent at index 0 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.70817894Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5302025-12-03T07:28:08.874Z{"msg":"extent 1 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3999, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.708246628Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5312025-12-03T07:28:08.874Z{"msg":"extent:1 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3999, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.708320246Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5322025-12-03T07:28:08.875Z{"msg":"found dest for source 0 for extent at index 1 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.708381526Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5332025-12-03T07:28:08.875Z{"msg":"extent 2 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3999, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.708442926Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5342025-12-03T07:28:08.875Z{"msg":"extent:2 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3999, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.708506496Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5352025-12-03T07:28:08.875Z{"msg":"found dest for source 0 for extent at index 2 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.708548909Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5362025-12-03T07:28:08.875Z{"msg":"extent 3 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3999, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.708604389Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5372025-12-03T07:28:08.875Z{"msg":"extent:3 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3999, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.708667699Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5382025-12-03T07:28:08.875Z{"msg":"found dest for source 0 for extent at index 3 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.708708812Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5392025-12-03T07:28:08.875Z{"msg":"extent 4 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3999, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.708788509Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5402025-12-03T07:28:08.878Z{"msg":"extent:4 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4000, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3999, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.708846339Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5412025-12-03T07:28:08.878Z{"msg":"found dest for source 0 for extent at index 4 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.708893721Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"mend","":"downstairs","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5422025-12-03T07:28:08.878Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.708945613Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"client":"0","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5432025-12-03T07:28:08.878Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.708989725Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"client":"1","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5442025-12-03T07:28:08.878Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.709030939Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"client":"2","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5452025-12-03T07:28:08.878Z{"msg":"Full repair list: {ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.709088389Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"downstairs","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5462025-12-03T07:28:08.878Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 4001, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 4001, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 4001, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 4001, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 4001, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.709389849Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"downstairs","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5472025-12-03T07:28:08.878Z{"msg":"starting reconciliation cc19028e-b7d6-4c7b-8249-f19835dccf56: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.70944402Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"downstairs","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5482025-12-03T07:28:08.878Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.709483764Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"client":"2","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5492025-12-03T07:28:08.884Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.719381865Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"client":"2","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5502025-12-03T07:28:08.888Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.72310061Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"client":"2","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5512025-12-03T07:28:08.891Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.726722601Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"client":"2","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5522025-12-03T07:28:08.894Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.730204714Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"client":"2","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5532025-12-03T07:28:08.898Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.733363931Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"downstairs","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5542025-12-03T07:28:08.898Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.733406414Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"client":"0","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5552025-12-03T07:28:08.898Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.733427771Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"client":"1","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5562025-12-03T07:28:08.898Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.733453007Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"client":"2","":"downstairs","session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5572025-12-03T07:28:08.901Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.733471673Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5582025-12-03T07:28:08.901Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.733501868Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5592025-12-03T07:28:08.901Z{"msg":"f7bb9a08-7346-4907-8a92-a50bd752c443 is now active with session: b8a44954-7f83-40a9-9e28-64e2efafc4a4","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.733538373Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"session_id":"b8a44954-7f83-40a9-9e28-64e2efafc4a4"}
5602025-12-03T07:28:08.901Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.733571647Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260}
5612025-12-03T07:28:08.901Z{"msg":"Activated sub_volume f7bb9a08-7346-4907-8a92-a50bd752c443","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.733591354Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260}
5622025-12-03T07:28:08.901Z{"msg":"f7bb9a08-7346-4907-8a92-a50bd752c443 active request set","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.733622739Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5632025-12-03T07:28:08.901Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.733653323Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260}
5642025-12-03T07:28:08.901Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.733679679Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"client":"0","":"downstairs","session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5652025-12-03T07:28:08.901Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.733711584Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"client":"1","":"downstairs","session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5662025-12-03T07:28:08.901Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.733745858Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"client":"2","":"downstairs","session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5672025-12-03T07:28:08.901Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.733778963Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5682025-12-03T07:28:08.901Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.733815527Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"io task","client":"0","":"downstairs","session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5692025-12-03T07:28:08.901Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.733850021Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"io task","client":"1","":"downstairs","session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5702025-12-03T07:28:08.901Z{"msg":"ds_connection connected from Ok(127.0.0.1:59314)","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.733887175Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5712025-12-03T07:28:08.902Z{"msg":"ds_connection connected from Ok(127.0.0.1:56112)","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.733922459Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"io task","client":"1","":"downstairs","session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5722025-12-03T07:28:08.902Z{"msg":"ds_connection connected from Ok(127.0.0.1:48473)","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.733954144Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"io task","client":"0","":"downstairs","session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5732025-12-03T07:28:08.902Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.734676694Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"client":"1","":"downstairs","session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5742025-12-03T07:28:08.902Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.734774078Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"client":"2","":"downstairs","session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5752025-12-03T07:28:08.902Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.73482271Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"client":"0","":"downstairs","session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5762025-12-03T07:28:08.902Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.734984783Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5772025-12-03T07:28:08.902Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.735139237Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5782025-12-03T07:28:08.902Z{"msg":"[0]R flush_numbers: [5321, 5321, 5321, 5321, 5321]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.735173572Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"downstairs","":"downstairs","session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5792025-12-03T07:28:08.902Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.735192559Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"downstairs","":"downstairs","session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5802025-12-03T07:28:08.902Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.735211196Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"downstairs","":"downstairs","session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5812025-12-03T07:28:08.902Z{"msg":"[1]R flush_numbers: [5321, 5321, 5321, 5321, 5321]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.735229463Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"downstairs","":"downstairs","session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5822025-12-03T07:28:08.902Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.73524646Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"downstairs","":"downstairs","session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5832025-12-03T07:28:08.902Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.735264317Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"downstairs","":"downstairs","session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5842025-12-03T07:28:08.902Z{"msg":"[2]R flush_numbers: [5321, 5321, 5321, 5321, 5321]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.735282214Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"downstairs","":"downstairs","session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5852025-12-03T07:28:08.902Z{"msg":"[2]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.73530638Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"downstairs","":"downstairs","session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5862025-12-03T07:28:08.902Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.735324717Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"downstairs","":"downstairs","session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5872025-12-03T07:28:08.902Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.735348943Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"downstairs","":"downstairs","session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5882025-12-03T07:28:08.902Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
5892025-12-03T07:28:08.902Z,"time":"2025-12-03T07:28:07.73536638Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"downstairs","":"downstairs","session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5902025-12-03T07:28:08.902Z{"msg":"Next flush: 5322","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.735390996Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"downstairs","":"downstairs","session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5912025-12-03T07:28:08.902Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.735414402Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"":"downstairs","":"downstairs","session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5922025-12-03T07:28:08.902Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.735431669Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5932025-12-03T07:28:08.902Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.735453705Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"client":"0","":"downstairs","session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5942025-12-03T07:28:08.902Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.735471063Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"client":"1","":"downstairs","session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5952025-12-03T07:28:08.902Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.735494319Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"client":"2","":"downstairs","session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5962025-12-03T07:28:08.902Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.735511526Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5972025-12-03T07:28:08.902Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.735533842Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5982025-12-03T07:28:08.902Z{"msg":"f7bb9a08-7346-4907-8a92-a50bd752c443 is now active with session: 4352b2ae-9828-475e-9443-bee84f3ea4fc","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.735552019Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
5992025-12-03T07:28:08.902Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.735573656Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260,"session_id":"4352b2ae-9828-475e-9443-bee84f3ea4fc"}
6002025-12-03T07:28:08.902Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.735589833Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260}
6012025-12-03T07:28:08.902Z{"msg":"Activated sub_volume f7bb9a08-7346-4907-8a92-a50bd752c443","v":0,"name":"crucible","level":30,"time":"2025-12-03T07:28:07.735603741Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260}
6022025-12-03T07:28:08.902ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
6032025-12-03T07:28:08.902ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
6042025-12-03T07:28:08.902ZRead and Verify all blocks (0..100 range:true)
6052025-12-03T07:28:08.989ZWrote out file "/var/tmp/test_up-build/verify_file"
6062025-12-03T07:28:08.989ZVerify test completed
6072025-12-03T07:28:08.992ZWrote out file "/var/tmp/test_up-build/verify_file"
6082025-12-03T07:28:08.992ZCLIENT: Tests done. All submitted work has been ACK'd
6092025-12-03T07:28:08.992Z----------------------------------------------------------------
6102025-12-03T07:28:08.992Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6112025-12-03T07:28:08.992ZStates: Active Active Active
6122025-12-03T07:28:08.992ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6132025-12-03T07:28:08.992Z 1000 Acked Read 50 Done Done Done false
6142025-12-03T07:28:08.992Z STATES DS:0 DS:1 DS:2 TOTAL
6152025-12-03T07:28:08.992Z Sent 0 0 0 0
6162025-12-03T07:28:08.992Z Done 1 1 1 3
6172025-12-03T07:28:08.992Z Skipped 0 0 0 0
6182025-12-03T07:28:08.992Z Error 0 0 0 0
6192025-12-03T07:28:08.992ZLast Flush: None None None
6202025-12-03T07:28:08.992ZDownstairs last five completed:
6212025-12-03T07:28:08.992ZUpstairs last five completed: 1000
6222025-12-03T07:28:08.992Z----------------------------------------------------------------
6232025-12-03T07:28:08.992Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6242025-12-03T07:28:08.992ZStates: Active Active Active
6252025-12-03T07:28:08.992ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6262025-12-03T07:28:08.992Z 1000 Acked Read 50 Done Done Done false
6272025-12-03T07:28:08.992Z STATES DS:0 DS:1 DS:2 TOTAL
6282025-12-03T07:28:08.992Z Sent 0 0 0 0
6292025-12-03T07:28:08.992Z Done 1 1 1 3
6302025-12-03T07:28:08.992Z Skipped 0 0 0 0
6312025-12-03T07:28:08.992Z Error 0 0 0 0
6322025-12-03T07:28:08.992ZLast Flush: None None None
6332025-12-03T07:28:08.992ZDownstairs last five completed:
6342025-12-03T07:28:08.992ZUpstairs last five completed: 1000
6352025-12-03T07:28:08.992ZCLIENT: Up:0 ds:2 act:6
6362025-12-03T07:28:12.992Z----------------------------------------------------------------
6372025-12-03T07:28:12.992Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6382025-12-03T07:28:12.992ZDownstairs last five completed: 1001 1000
6392025-12-03T07:28:12.992ZUpstairs last five completed: 1001 1000
6402025-12-03T07:28:12.995Z----------------------------------------------------------------
6412025-12-03T07:28:12.995Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6422025-12-03T07:28:12.995ZDownstairs last five completed: 1001 1000
6432025-12-03T07:28:12.995ZUpstairs last five completed: 1001 1000
6442025-12-03T07:28:12.995ZCLIENT: Up:0 ds:0 act:6
6452025-12-03T07:28:12.995ZCLIENT: All crucible jobs finished, exiting program
6462025-12-03T07:28:12.995Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-12-03T07:28:11.826729119Z","hostname":"w-01KBHHBBBJXNE4VXXX13943CNR","pid":1260}
6472025-12-03T07:28:12.999ZRepair part 2 passed
6482025-12-03T07:28:13.002Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
6492025-12-03T07:28:13.018ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6502025-12-03T07:28:13.018Z 0 000-009 32 32 32 4000 4000 4000 F F F
6512025-12-03T07:28:13.020Z 1 010-019 32 32 32 4000 4000 4000 F F F
6522025-12-03T07:28:13.020Z 2 020-029 32 32 32 4000 4000 4000 F F F
6532025-12-03T07:28:13.021Z 3 030-039 32 32 32 4000 4000 4000 F F F
6542025-12-03T07:28:13.021Z 4 040-049 32 32 32 4000 4000 4000 F F F
6552025-12-03T07:28:13.021ZMax gen: 32, Max flush: 4000
6562025-12-03T07:28:13.023Zdump test passed
6572025-12-03T07:28:13.023Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
6582025-12-03T07:28:13.039Z Extent 1
6592025-12-03T07:28:13.042ZGEN 32 32 32
6602025-12-03T07:28:13.042ZFLUSH_ID 4000 4000 4000
6612025-12-03T07:28:13.042ZDIRTY
6622025-12-03T07:28:13.043Z
6632025-12-03T07:28:13.043ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
6642025-12-03T07:28:13.043Z010 A A A A A A
6652025-12-03T07:28:13.045Z011 A A A A A A
6662025-12-03T07:28:13.045Z012 A A A A A A
6672025-12-03T07:28:13.048Z013 A A A A A A
6682025-12-03T07:28:13.048Z014 A A A A A A
6692025-12-03T07:28:13.051Z015 A A A A A A
6702025-12-03T07:28:13.051Z016 A A A A A A
6712025-12-03T07:28:13.054Z017 A A A A A A
6722025-12-03T07:28:13.054Z018 A A A A A A
6732025-12-03T07:28:13.057Z019 A A A A A A
6742025-12-03T07:28:13.059Zdump extent test passed
6752025-12-03T07:28:13.059Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
6762025-12-03T07:28:13.076ZExtent 2 Block in extent 0 Actual block 20
6772025-12-03T07:28:13.079Z
6782025-12-03T07:28:13.079Z DATA SHA256 VER
6792025-12-03T07:28:13.079Z------ ---------------------------------------------------------------- ---
6802025-12-03T07:28:13.079Z 0 ea2f414db2388712665fb05638280903bb7ff6011b8dd8746c0ca50fff1328a3 A
6812025-12-03T07:28:13.082Z 1 ea2f414db2388712665fb05638280903bb7ff6011b8dd8746c0ca50fff1328a3 A
6822025-12-03T07:28:13.082Z 2 ea2f414db2388712665fb05638280903bb7ff6011b8dd8746c0ca50fff1328a3 A
6832025-12-03T07:28:13.082Z
6842025-12-03T07:28:13.082ZNONCES 0 1 2 DIFF
6852025-12-03T07:28:13.082Z------ ------------------------ ------------------------ ------------------------ -----
6862025-12-03T07:28:13.082Z 0 492af1ccb2f81527cf2c4a9f 492af1ccb2f81527cf2c4a9f 492af1ccb2f81527cf2c4a9f
6872025-12-03T07:28:13.082Z
6882025-12-03T07:28:13.082Z TAGS 0 1 2 DIFF
6892025-12-03T07:28:13.082Z------ -------------------------------- -------------------------------- -------------------------------- -----
6902025-12-03T07:28:13.082Z 0 c284e429f8e4d9c4aa0a10d908ffaa5b c284e429f8e4d9c4aa0a10d908ffaa5b c284e429f8e4d9c4aa0a10d908ffaa5b
6912025-12-03T07:28:13.082Z
6922025-12-03T07:28:13.082ZHASHES 0 1 2 DIFF
6932025-12-03T07:28:13.082Z------ ---------------- ---------------- ---------------- -----
6942025-12-03T07:28:13.082Z
6952025-12-03T07:28:13.085Zdump block test passed
6962025-12-03T07:28:13.085ZInitial upstairs tests have completed, stopping all downstairs
6972025-12-03T07:28:18.151ZCreating 7 larger downstairs regions
6982025-12-03T07:28:18.514ZStarting 7 downstairs
6992025-12-03T07:28:23.523Zdsc restarted at PID: 1276
7002025-12-03T07:28:23.934ZNow do the replace-reconcile test
7012025-12-03T07:28:23.990ZUsing 8870 for the replacement port
7022025-12-03T07:31:55.696ZCompleted test: replace-reconcile
7032025-12-03T07:31:55.699ZNow do the replace-before-active test
7042025-12-03T07:38:02.872ZCompleted test: replace-before-active
7052025-12-03T07:38:02.876ZAll tests have completed, stopping all downstairs
7062025-12-03T07:38:02.933Z
7072025-12-03T07:38:02.933ZAll Tests have passed
7082025-12-03T07:38:02.937Z16:01 Test duration
7092025-12-03T07:38:02.937Z
7102025-12-03T07:38:02.937Zreal 16:01.072288732
7112025-12-03T07:38:02.937Zuser 40:54.950313717
7122025-12-03T07:38:02.937Zsys 32.112594439
7132025-12-03T07:38:02.937Ztrap 1.394535229
7142025-12-03T07:38:02.937Ztflt 0.053514185
7152025-12-03T07:38:02.937Zdflt 0.050571640
7162025-12-03T07:38:02.937Zkflt 0.001245797
7172025-12-03T07:38:02.937Zlock 23:49:05.924885699
7182025-12-03T07:38:02.937Zslp 1:13:21.051935903
7192025-12-03T07:38:02.937Zlat 5:06.394039994
7202025-12-03T07:38:02.937Zstop 0.048749697
7212025-12-03T07:38:02.937Z+ echo 'test-up-2region-encrypted ends'
7222025-12-03T07:38:02.937Ztest-up-2region-encrypted ends
7232025-12-03T07:38:07.940Zprocess exited: duration 971445 ms, exit code 0
7242025-12-03T07:38:07.940Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
7252025-12-03T07:39:07.987Zexec warning: : stdout descriptor may be held open by a background process; giving up!
7262025-12-03T07:39:07.987Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
7272025-12-03T07:39:07.999Zfound 17 output files
7282025-12-03T07:39:07.999Zuploading: /tmp/test_up-build/dsc-out.txt (54721 bytes)
7292025-12-03T07:39:09.012Zuploaded: /tmp/test_up-build/dsc-out.txt
7302025-12-03T07:39:09.021Zuploading: /tmp/test_up-build/test_up_out.txt (21209830 bytes)
7312025-12-03T07:39:10.264Zuploaded: /tmp/test_up-build/test_up_out.txt
7322025-12-03T07:39:10.267Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (550940 bytes)
7332025-12-03T07:39:11.285Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
7342025-12-03T07:39:11.288Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (548364 bytes)
7352025-12-03T07:39:12.308Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
7362025-12-03T07:39:12.308Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (979538 bytes)
7372025-12-03T07:39:13.332Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
7382025-12-03T07:39:13.336Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (543085 bytes)
7392025-12-03T07:39:14.353Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
7402025-12-03T07:39:14.356Zuploading: /tmp/test_up-build/dsc/downstairs-8850.txt (1341296 bytes)
7412025-12-03T07:39:15.381Zuploaded: /tmp/test_up-build/dsc/downstairs-8850.txt
7422025-12-03T07:39:15.385Zuploading: /tmp/test_up-build/dsc/downstairs-8860.txt (42054 bytes)
7432025-12-03T07:39:16.395Zuploaded: /tmp/test_up-build/dsc/downstairs-8860.txt
7442025-12-03T07:39:16.398Zuploading: /tmp/test_up-build/dsc/downstairs-8870.txt (3705135 bytes)
7452025-12-03T07:39:17.448Zuploaded: /tmp/test_up-build/dsc/downstairs-8870.txt
7462025-12-03T07:39:17.448Zuploading: /tmp/debug/df.txt (1270 bytes)
7472025-12-03T07:39:18.457Zuploaded: /tmp/debug/df.txt
7482025-12-03T07:39:18.460Zuploading: /tmp/debug/dtrace.txt (1426904 bytes)
7492025-12-03T07:39:19.487Zuploaded: /tmp/debug/dtrace.txt
7502025-12-03T07:39:19.490Zuploading: /tmp/debug/iostat.txt (309381 bytes)
7512025-12-03T07:39:19.499Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 309381 -> 312692
7522025-12-03T07:39:20.506Zuploaded: /tmp/debug/iostat.txt
7532025-12-03T07:39:20.510Zuploading: /tmp/debug/mpstat.txt (749736 bytes)
7542025-12-03T07:39:20.528Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 749736 -> 758496
7552025-12-03T07:39:21.535Zuploaded: /tmp/debug/mpstat.txt
7562025-12-03T07:39:21.538Zuploading: /tmp/debug/paging.txt (135406 bytes)
7572025-12-03T07:39:21.546Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 135406 -> 137072
7582025-12-03T07:39:22.553Zuploaded: /tmp/debug/paging.txt
7592025-12-03T07:39:22.553Zuploading: /tmp/debug/prstat.txt (1481650 bytes)
7602025-12-03T07:39:22.577Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 1481650 -> 1501077
7612025-12-03T07:39:23.585Zuploaded: /tmp/debug/prstat.txt
7622025-12-03T07:39:23.585Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
7632025-12-03T07:39:24.595Zuploaded: /tmp/debug/psrinfo.txt
7642025-12-03T07:39:24.595Zuploading: /tmp/debug/upstairs-info.txt (231217 bytes)
7652025-12-03T07:39:25.611Zuploaded: /tmp/debug/upstairs-info.txt