01JHJRD020MPTW1S44DWH0T66M: test-up-2region-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01JHJRD9YNARDN08YN8CNV36FQ

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12025-01-14T16:09:13.387Zjob dependencies complete; ready to run (waiting for 14 m 2 s)
22025-01-14T16:10:03.582Zjob assigned to worker 01JHJS779VQF5PA1M2YNG80QCD [factory aws, i-0ae76c408d082e359] (queued for 50 s)
32025-01-14T16:10:11.492Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42025-01-14T16:10:13.293Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52025-01-14T16:10:13.293Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62025-01-14T16:10:15.539Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72025-01-14T16:10:15.539Zdownloading input: /input/build/work/bins/crutest.gz
82025-01-14T16:10:18.257Zdownloaded input: /input/build/work/bins/crutest.gz
92025-01-14T16:10:18.257Zdownloading input: /input/build/work/bins/dsc.gz
102025-01-14T16:10:19.598Zdownloaded input: /input/build/work/bins/dsc.gz
112025-01-14T16:10:19.603Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122025-01-14T16:10:19.621Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132025-01-14T16:10:19.621Zdownloading input: /input/build/work/scripts/test_ds.sh
142025-01-14T16:10:19.639Zdownloaded input: /input/build/work/scripts/test_ds.sh
152025-01-14T16:10:19.639Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162025-01-14T16:10:19.658Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172025-01-14T16:10:19.658Zdownloading input: /input/build/work/scripts/test_repair.sh
182025-01-14T16:10:19.684Zdownloaded input: /input/build/work/scripts/test_repair.sh
192025-01-14T16:10:19.684Zdownloading input: /input/build/work/scripts/test_replay.sh
202025-01-14T16:10:19.706Zdownloaded input: /input/build/work/scripts/test_replay.sh
212025-01-14T16:10:19.706Zdownloading input: /input/build/work/scripts/test_up.sh
222025-01-14T16:10:19.725Zdownloaded input: /input/build/work/scripts/test_up.sh
232025-01-14T16:10:19.725Zdownloading input: /input/build/work/scripts/upstairs_info.d
242025-01-14T16:10:19.748Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252025-01-14T16:10:19.753Zdownloading input: /input/build/tmp/cargo-test-out.log
262025-01-14T16:10:21.104Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272025-01-14T16:10:21.104Zstarting task 0: "setup"
282025-01-14T16:10:21.113Z++ uname -s
292025-01-14T16:10:21.118Z+ kern=SunOS
302025-01-14T16:10:21.118Z+ case "$kern" in
312025-01-14T16:10:21.118Z+ groupadd -g 12345 build
322025-01-14T16:10:21.118Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332025-01-14T16:10:23.122Z+ zfs create -o mountpoint=/work rpool/work
342025-01-14T16:10:23.303Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352025-01-14T16:10:23.306Z+ home_fs=zfs
362025-01-14T16:10:23.306Z+ [[ zfs == autofs ]]
372025-01-14T16:10:23.306Z+ mkdir -p /home/build
382025-01-14T16:10:23.309Z+ chown build:build /home/build /work
392025-01-14T16:10:25.308Z+ chmod 0700 /home/build /work
402025-01-14T16:10:25.311Zprocess exited: duration 4211 ms, exit code 0
 
412025-01-14T16:10:25.316Zstarting task 1: "authentication"
422025-01-14T16:10:25.333Zprocess exited: duration 16 ms, exit code 0
 
432025-01-14T16:10:25.337Zstarting task 2: "build"
442025-01-14T16:10:25.342Z+ banner cores
452025-01-14T16:10:25.345Z
462025-01-14T16:10:25.345Z #### #### ##### ###### ####
472025-01-14T16:10:25.345Z # # # # # # # #
482025-01-14T16:10:25.345Z # # # # # ##### ####
492025-01-14T16:10:25.348Z # # # ##### # #
502025-01-14T16:10:25.348Z # # # # # # # # #
512025-01-14T16:10:25.348Z #### #### # # ###### ####
522025-01-14T16:10:25.348Z
532025-01-14T16:10:25.348Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542025-01-14T16:10:25.354Z+ banner unpack
552025-01-14T16:10:25.357Z
562025-01-14T16:10:25.357Z # # # # ##### ## #### # #
572025-01-14T16:10:25.357Z # # ## # # # # # # # # #
582025-01-14T16:10:25.357Z # # # # # # # # # # ####
592025-01-14T16:10:25.357Z # # # # # ##### ###### # # #
602025-01-14T16:10:25.357Z # # # ## # # # # # # #
612025-01-14T16:10:25.357Z #### # # # # # #### # #
622025-01-14T16:10:25.358Z
632025-01-14T16:10:25.358Z+ mkdir -p /var/tmp/bins
642025-01-14T16:10:25.360Z+ for t in "$input/bins/"*.gz
652025-01-14T16:10:25.360Z++ basename /input/build/work/bins/crucible-downstairs.gz
662025-01-14T16:10:25.360Z+ b=crucible-downstairs.gz
672025-01-14T16:10:25.360Z+ b=crucible-downstairs
682025-01-14T16:10:25.360Z+ gunzip
692025-01-14T16:10:27.612Z+ chmod +x /var/tmp/bins/crucible-downstairs
702025-01-14T16:10:27.615Z+ for t in "$input/bins/"*.gz
712025-01-14T16:10:27.615Z++ basename /input/build/work/bins/crucible-hammer.gz
722025-01-14T16:10:27.618Z+ b=crucible-hammer.gz
732025-01-14T16:10:27.618Z+ b=crucible-hammer
742025-01-14T16:10:27.618Z+ gunzip
752025-01-14T16:10:29.748Z+ chmod +x /var/tmp/bins/crucible-hammer
762025-01-14T16:10:29.751Z+ for t in "$input/bins/"*.gz
772025-01-14T16:10:29.754Z++ basename /input/build/work/bins/crutest.gz
782025-01-14T16:10:29.754Z+ b=crutest.gz
792025-01-14T16:10:29.754Z+ b=crutest
802025-01-14T16:10:29.754Z+ gunzip
812025-01-14T16:10:32.119Z+ chmod +x /var/tmp/bins/crutest
822025-01-14T16:10:32.123Z+ for t in "$input/bins/"*.gz
832025-01-14T16:10:32.126Z++ basename /input/build/work/bins/dsc.gz
842025-01-14T16:10:32.126Z+ b=dsc.gz
852025-01-14T16:10:32.126Z+ b=dsc
862025-01-14T16:10:32.126Z+ gunzip
872025-01-14T16:10:33.491Z+ chmod +x /var/tmp/bins/dsc
882025-01-14T16:10:33.493Z+ export BINDIR=/var/tmp/bins
892025-01-14T16:10:33.493Z+ BINDIR=/var/tmp/bins
902025-01-14T16:10:33.493Z+ jobpid=1141
912025-01-14T16:10:33.496Z+ echo 'Setup debug logging'
922025-01-14T16:10:33.496ZSetup debug logging
932025-01-14T16:10:33.496Z+ mkdir /tmp/debug
942025-01-14T16:10:33.496Z+ sleep 3600
952025-01-14T16:10:33.496Z+ psrinfo -v
962025-01-14T16:10:33.499Z+ df -h
972025-01-14T16:10:33.506Z+ prstat -d d -mLc 1
982025-01-14T16:10:33.510Z+ iostat -T d -xn 1
992025-01-14T16:10:33.510Z+ mpstat -T d 1
1002025-01-14T16:10:33.510Z+ vmstat -T d -p 1
1012025-01-14T16:10:33.513Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1022025-01-14T16:10:33.513Z+ banner test_up_2r_encrypted
1032025-01-14T16:10:33.513Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1042025-01-14T16:10:33.513Z #####
1052025-01-14T16:10:33.513Z ##### ###### #### ##### # # ##### # # #####
1062025-01-14T16:10:33.513Z # # # # # # # # # # #
1072025-01-14T16:10:33.514Z # ##### #### # # # # # ##### # #
1082025-01-14T16:10:33.514Z # # # # # # ##### # #####
1092025-01-14T16:10:33.514Z # # # # # # # # # # #
1102025-01-14T16:10:33.514Z # ###### #### # ####### #### # ####### ####### # #
1112025-01-14T16:10:33.514Z
1122025-01-14T16:10:33.514Z+ ptime -m bash /input/build/work/scripts/test_up.sh -r 2 -N encrypted
1132025-01-14T16:10:33.522Z/input/build/work
1142025-01-14T16:10:33.533ZUsing 2 region sets
1152025-01-14T16:10:33.533ZTurn off color for downstairs dump
1162025-01-14T16:10:33.548ZUpstairs using key: 93Moh3MNIMkYzI46QoVAVPKhai1YPtEopcGQw+wY/3w=
1172025-01-14T16:10:33.551Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1182025-01-14T16:10:33.551ZCreating 6 downstairs regions
1192025-01-14T16:10:33.920ZStarting 6 downstairs
1202025-01-14T16:10:33.922Zdsc started at PID: 1189
1212025-01-14T16:10:39.247ZDisable automatic restart on all downstairs
1222025-01-14T16:10:39.298Z
1232025-01-14T16:10:39.298ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1242025-01-14T16:10:39.302ZRunning test: span
1252025-01-14T16:10:43.468ZCompleted test: span
1262025-01-14T16:10:48.475ZRunning test: big
1272025-01-14T16:10:53.047ZCompleted test: big
1282025-01-14T16:10:58.054ZRunning test: dep
1292025-01-14T16:11:31.554ZCompleted test: dep
1302025-01-14T16:11:36.561ZRunning test: balloon
1312025-01-14T16:15:48.545ZCompleted test: balloon
1322025-01-14T16:15:53.551ZRunning test: deactivate
1332025-01-14T16:16:48.187ZCompleted test: deactivate
1342025-01-14T16:16:53.193ZRunning hammer
1352025-01-14T16:17:10.852ZRun repair tests
1362025-01-14T16:17:10.855Z/var/tmp/bins/crutest fill -g 31 -q --key 93Moh3MNIMkYzI46QoVAVPKhai1YPtEopcGQw+wY/3w= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1372025-01-14T16:17:10.913Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.480976851Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"task":"crutest"}
1382025-01-14T16:17:10.916Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.481577628Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"task":"crutest"}
1392025-01-14T16:17:10.916Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.483172877Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"task":"crutest"}
1402025-01-14T16:17:10.916Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.483354309Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232}
1412025-01-14T16:17:10.916Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.48343478Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232}
1422025-01-14T16:17:10.916Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.483470971Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232}
1432025-01-14T16:17:10.916Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.483488341Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232}
1442025-01-14T16:17:10.916Z{"msg":"Crucible afbaa53f-8339-4d8d-9c73-314e93b4f161 has session id: aa25e079-60a8-4d75-a5c0-cea40f28c9d4","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.484008467Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1452025-01-14T16:17:10.916Z{"msg":"Upstairs opts: Upstairs UUID: afbaa53f-8339-4d8d-9c73-314e93b4f161, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.484061858Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1462025-01-14T16:17:10.916Z{"msg":"Crucible stats registered with UUID: afbaa53f-8339-4d8d-9c73-314e93b4f161","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.484085648Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1472025-01-14T16:17:10.916Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:10.484099438Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1482025-01-14T16:17:10.918Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.486008141Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"task":"crutest"}
1492025-01-14T16:17:10.919Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.486073822Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232}
1502025-01-14T16:17:10.919Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.486108542Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232}
1512025-01-14T16:17:10.919Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.486128373Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232}
1522025-01-14T16:17:10.919Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.486142613Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232}
1532025-01-14T16:17:10.919Z{"msg":"Crucible afbaa53f-8339-4d8d-9c73-314e93b4f161 has session id: 0a2efd4d-2700-43c4-959f-f4e7b1dab7ec","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.486590718Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
1542025-01-14T16:17:10.919Z{"msg":"Upstairs opts: Upstairs UUID: afbaa53f-8339-4d8d-9c73-314e93b4f161, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.486633969Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
1552025-01-14T16:17:10.919Z{"msg":"Crucible stats registered with UUID: afbaa53f-8339-4d8d-9c73-314e93b4f161","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.486656419Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
1562025-01-14T16:17:10.919Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:10.48670664Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
1572025-01-14T16:17:10.923Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.487378208Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"task":"crutest"}
1582025-01-14T16:17:10.923Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.487436558Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232}
1592025-01-14T16:17:10.923Z{"msg":"afbaa53f-8339-4d8d-9c73-314e93b4f161 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.487476299Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1602025-01-14T16:17:10.923Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.487495739Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"client":"0","":"downstairs","session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1612025-01-14T16:17:10.923Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.487536579Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"client":"0","":"downstairs","session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1622025-01-14T16:17:10.923Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.48755429Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"client":"1","":"downstairs","session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1632025-01-14T16:17:10.923Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.48759359Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"client":"1","":"downstairs","session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1642025-01-14T16:17:10.923Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.48762626Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"client":"2","":"downstairs","session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1652025-01-14T16:17:10.923Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.487651911Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"client":"2","":"downstairs","session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1662025-01-14T16:17:10.923Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.487696681Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1672025-01-14T16:17:10.923Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.487723142Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1682025-01-14T16:17:10.923Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.487757752Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1692025-01-14T16:17:10.923Z{"msg":"ds_connection connected from Ok(127.0.0.1:63588)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.487792333Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1702025-01-14T16:17:10.923Z{"msg":"ds_connection connected from Ok(127.0.0.1:56451)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.487814373Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1712025-01-14T16:17:10.923Z{"msg":"ds_connection connected from Ok(127.0.0.1:49214)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.487836333Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1722025-01-14T16:17:10.924Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.488586672Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"client":"0","":"downstairs","session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1732025-01-14T16:17:10.924Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.488628443Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"client":"2","":"downstairs","session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1742025-01-14T16:17:10.924Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.488657773Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"client":"1","":"downstairs","session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1752025-01-14T16:17:10.924Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.489203979Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"client":"0","":"downstairs","session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1762025-01-14T16:17:10.924Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.48926233Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"client":"1","":"downstairs","session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1772025-01-14T16:17:10.924Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.489293651Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"client":"2","":"downstairs","session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1782025-01-14T16:17:10.924Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.489619784Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1792025-01-14T16:17:10.924Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.489656165Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1802025-01-14T16:17:10.924Z{"msg":"[0]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.489672255Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"downstairs","":"downstairs","session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1812025-01-14T16:17:10.924Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.489703316Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"downstairs","":"downstairs","session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1822025-01-14T16:17:10.924Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.489725896Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"downstairs","":"downstairs","session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1832025-01-14T16:17:10.924Z{"msg":"[1]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.489740106Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"downstairs","":"downstairs","session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1842025-01-14T16:17:10.924Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.489753316Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"downstairs","":"downstairs","session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1852025-01-14T16:17:10.924Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.489774416Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"downstairs","":"downstairs","session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1862025-01-14T16:17:10.924Z{"msg":"[2]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.489788077Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"downstairs","":"downstairs","session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1872025-01-14T16:17:10.924Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.489801137Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"downstairs","":"downstairs","session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1882025-01-14T16:17:10.924Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.489821027Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"downstairs","":"downstairs","session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1892025-01-14T16:17:10.924Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.489834257Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"downstairs","":"downstairs","session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1902025-01-14T16:17:10.924Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.489847057Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"downstairs","":"downstairs","session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1912025-01-14T16:17:10.924Z{"msg":"Next flush: 3999","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.489866287Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"downstairs","":"downstairs","session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1922025-01-14T16:17:10.924Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.489879187Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"downstairs","":"downstairs","session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1932025-01-14T16:17:10.924Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.489892018Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1942025-01-14T16:17:10.924Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.489915958Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"client":"0","":"downstairs","session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1952025-01-14T16:17:10.924Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.489929248Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"client":"1","":"downstairs","session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1962025-01-14T16:17:10.924Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.489942128Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"client":"2","":"downstairs","session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1972025-01-14T16:17:10.924Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.489955088Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1982025-01-14T16:17:10.924Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.489967179Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
1992025-01-14T16:17:10.924Z{"msg":"afbaa53f-8339-4d8d-9c73-314e93b4f161 is now active with session: aa25e079-60a8-4d75-a5c0-cea40f28c9d4","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.489987699Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
2002025-01-14T16:17:10.924Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.490028959Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"session_id":"aa25e079-60a8-4d75-a5c0-cea40f28c9d4"}
2012025-01-14T16:17:10.924Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.49007248Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232}
2022025-01-14T16:17:10.924Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.4901075Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232}
2032025-01-14T16:17:10.924Z{"msg":"afbaa53f-8339-4d8d-9c73-314e93b4f161 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.490307933Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2042025-01-14T16:17:10.924Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.490342763Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"client":"0","":"downstairs","session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2052025-01-14T16:17:10.924Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.490369504Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"client":"0","":"downstairs","session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2062025-01-14T16:17:10.924Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.490406724Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"client":"1","":"downstairs","session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2072025-01-14T16:17:10.924Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.490438654Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"client":"1","":"downstairs","session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2082025-01-14T16:17:10.924Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.490467185Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"client":"2","":"downstairs","session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2092025-01-14T16:17:10.925Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.490494755Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"client":"2","":"downstairs","session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2102025-01-14T16:17:10.925Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.490521045Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2112025-01-14T16:17:10.925Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.490546606Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2122025-01-14T16:17:10.925Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.490561626Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2132025-01-14T16:17:10.925Z{"msg":"ds_connection connected from Ok(127.0.0.1:48376)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.490581516Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2142025-01-14T16:17:10.925Z{"msg":"ds_connection connected from Ok(127.0.0.1:51010)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.490605036Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2152025-01-14T16:17:10.925Z{"msg":"ds_connection connected from Ok(127.0.0.1:54160)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.490630087Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2162025-01-14T16:17:10.925Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.490662757Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"client":"1","":"downstairs","session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2172025-01-14T16:17:10.925Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.490688047Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"client":"2","":"downstairs","session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2182025-01-14T16:17:10.928Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.490747568Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"client":"0","":"downstairs","session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2192025-01-14T16:17:10.928Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.491144113Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"client":"1","":"downstairs","session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2202025-01-14T16:17:10.928Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.491230854Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"client":"2","":"downstairs","session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2212025-01-14T16:17:10.928Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.491315125Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"client":"0","":"downstairs","session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2222025-01-14T16:17:10.928Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.491541818Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2232025-01-14T16:17:10.928Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.491590628Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2242025-01-14T16:17:10.928Z{"msg":"[0]R flush_numbers: [5319, 5319, 5319, 5319, 5317]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.49170394Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"downstairs","":"downstairs","session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2252025-01-14T16:17:10.928Z{"msg":"[0]R generation: [14, 14, 14, 14, 13]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.4917401Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"downstairs","":"downstairs","session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2262025-01-14T16:17:10.928Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.49175613Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"downstairs","":"downstairs","session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2272025-01-14T16:17:10.928Z{"msg":"[1]R flush_numbers: [5319, 5319, 5319, 5319, 5317]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.49177847Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"downstairs","":"downstairs","session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2282025-01-14T16:17:10.928Z{"msg":"[1]R generation: [14, 14, 14, 14, 13]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.491792061Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"downstairs","":"downstairs","session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2292025-01-14T16:17:10.930Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.491812361Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"downstairs","":"downstairs","session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2302025-01-14T16:17:10.930Z{"msg":"[2]R flush_numbers: [5319, 5319, 5319, 5319, 5317]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.491825711Z","hostname":"Wait for a query_work_queue command to finish before sending IO
2312025-01-14T16:17:10.930Zw-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"downstairs","":"downstairs","session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2322025-01-14T16:17:10.930Z{"msg":"[2]R generation: [14, 14, 14, 14, 13]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.491854431Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"downstairs","":"downstairs","session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2332025-01-14T16:17:10.930Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.491867511Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"downstairs","":"downstairs","session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2342025-01-14T16:17:10.930Z{"msg":"Max found gen is 15","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.491887182Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"downstairs","":"downstairs","session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2352025-01-14T16:17:10.930Z{"msg":"Generation requested: 31 >= found:15","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.491901132Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"downstairs","":"downstairs","session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2362025-01-14T16:17:10.930Z{"msg":"Next flush: 5320","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.491920892Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"downstairs","":"downstairs","session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2372025-01-14T16:17:10.930Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.491933752Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"":"downstairs","":"downstairs","session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2382025-01-14T16:17:10.930Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.491946452Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2392025-01-14T16:17:10.930Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.491965652Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"client":"0","":"downstairs","session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2402025-01-14T16:17:10.930Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.491978853Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"client":"1","":"downstairs","session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2412025-01-14T16:17:10.930Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.492012463Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"client":"2","":"downstairs","session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2422025-01-14T16:17:10.930Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.492027033Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2432025-01-14T16:17:10.931Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.492046413Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2442025-01-14T16:17:10.931Z{"msg":"afbaa53f-8339-4d8d-9c73-314e93b4f161 is now active with session: 0a2efd4d-2700-43c4-959f-f4e7b1dab7ec","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.492059364Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2452025-01-14T16:17:10.931Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.492071174Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232,"session_id":"0a2efd4d-2700-43c4-959f-f4e7b1dab7ec"}
2462025-01-14T16:17:10.931Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:10.492090234Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232}
2472025-01-14T16:17:10.931ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
2482025-01-14T16:17:10.931ZFill test
2492025-01-14T16:17:10.966ZRead and Verify all blocks (0..100 range:false)
2502025-01-14T16:17:11.045ZWrote out file "/var/tmp/test_up-build/verify_file"
2512025-01-14T16:17:11.046ZCLIENT: Tests done. All submitted work has been ACK'd
2522025-01-14T16:17:11.048Z----------------------------------------------------------------
2532025-01-14T16:17:11.048Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2542025-01-14T16:17:11.048ZStates: Active Active Active
2552025-01-14T16:17:11.048ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2562025-01-14T16:17:11.048Z 1002 Acked Read 50 Done Done Done false
2572025-01-14T16:17:11.048Z STATES DS:0 DS:1 DS:2 TOTAL
2582025-01-14T16:17:11.048Z Sent 0 0 0 0
2592025-01-14T16:17:11.048Z Done 1 1 1 3
2602025-01-14T16:17:11.048Z Skipped 0 0 0 0
2612025-01-14T16:17:11.048Z Error 0 0 0 0
2622025-01-14T16:17:11.048ZLast Flush: 1001 1001 1001
2632025-01-14T16:17:11.048ZDownstairs last five completed: 1001 1000
2642025-01-14T16:17:11.048ZUpstairs last five completed: 1002 1001 1000
2652025-01-14T16:17:11.049Z----------------------------------------------------------------
2662025-01-14T16:17:11.049Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2672025-01-14T16:17:11.049ZStates: Active Active Active
2682025-01-14T16:17:11.049ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2692025-01-14T16:17:11.049Z 1002 Acked Read 50 Done Done Done false
2702025-01-14T16:17:11.049Z STATES DS:0 DS:1 DS:2 TOTAL
2712025-01-14T16:17:11.049Z Sent 0 0 0 0
2722025-01-14T16:17:11.049Z Done 1 1 1 3
2732025-01-14T16:17:11.049Z Skipped 0 0 0 0
2742025-01-14T16:17:11.049Z Error 0 0 0 0
2752025-01-14T16:17:11.049ZLast Flush: 1001 1001 1001
2762025-01-14T16:17:11.049ZDownstairs last five completed: 1001 1000
2772025-01-14T16:17:11.049ZUpstairs last five completed: 1002 1001 1000
2782025-01-14T16:17:11.049ZCLIENT: Up:0 ds:2 act:6
2792025-01-14T16:17:15.047Z----------------------------------------------------------------
2802025-01-14T16:17:15.047Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2812025-01-14T16:17:15.047ZDownstairs last five completed: 1003 1002 1001 1000
2822025-01-14T16:17:15.047ZUpstairs last five completed: 1003 1002 1001 1000
2832025-01-14T16:17:15.047Z----------------------------------------------------------------
2842025-01-14T16:17:15.047Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2852025-01-14T16:17:15.047ZDownstairs last five completed: 1003 1002 1001 1000
2862025-01-14T16:17:15.048ZUpstairs last five completed: 1003 1002 1001 1000
2872025-01-14T16:17:15.051ZCLIENT: Up:0 ds:0 act:6
2882025-01-14T16:17:15.051ZCLIENT: All crucible jobs finished, exiting program
2892025-01-14T16:17:15.051Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:14.615760474Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1232}
2902025-01-14T16:17:15.058ZRepair setup passed
2912025-01-14T16:17:15.061ZCopy the region for /var/tmp/test_up-build/8830
2922025-01-14T16:17:15.063Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2932025-01-14T16:17:15.066Z/var/tmp/bins/crutest fill -g 32 -q --key 93Moh3MNIMkYzI46QoVAVPKhai1YPtEopcGQw+wY/3w= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2942025-01-14T16:17:15.122Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.689998995Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"task":"crutest"}
2952025-01-14T16:17:15.124Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.690654693Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"task":"crutest"}
2962025-01-14T16:17:15.124Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.692286912Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"task":"crutest"}
2972025-01-14T16:17:15.124Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.692450914Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238}
2982025-01-14T16:17:15.124Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.692496065Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238}
2992025-01-14T16:17:15.124Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.692519045Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238}
3002025-01-14T16:17:15.124Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.692533985Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238}
3012025-01-14T16:17:15.128Z{"msg":"Crucible f5245e69-5477-40fa-8e61-fea352b2811a has session id: eee38f73-1d48-47dd-95fa-4e7d6291c2f0","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.693115482Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3022025-01-14T16:17:15.128Z{"msg":"Upstairs opts: Upstairs UUID: f5245e69-5477-40fa-8e61-fea352b2811a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.693165413Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3032025-01-14T16:17:15.128Z{"msg":"Crucible stats registered with UUID: f5245e69-5477-40fa-8e61-fea352b2811a","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.693192573Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3042025-01-14T16:17:15.128Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:14.693224054Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3052025-01-14T16:17:15.128Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.695136306Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"task":"crutest"}
3062025-01-14T16:17:15.128Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.695218138Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238}
3072025-01-14T16:17:15.128Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.695253368Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238}
3082025-01-14T16:17:15.128Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.695274008Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238}
3092025-01-14T16:17:15.128Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.695288878Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238}
3102025-01-14T16:17:15.128Z{"msg":"Crucible f5245e69-5477-40fa-8e61-fea352b2811a has session id: 2dac29f4-e938-4d00-aada-e4bdeb504a10","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.695665743Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3112025-01-14T16:17:15.128Z{"msg":"Upstairs opts: Upstairs UUID: f5245e69-5477-40fa-8e61-fea352b2811a, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.695707583Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3122025-01-14T16:17:15.128Z{"msg":"Crucible stats registered with UUID: f5245e69-5477-40fa-8e61-fea352b2811a","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.695732144Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3132025-01-14T16:17:15.128Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:14.695773404Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3142025-01-14T16:17:15.132Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.696366131Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"task":"crutest"}
3152025-01-14T16:17:15.132Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.696402572Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238}
3162025-01-14T16:17:15.132Z{"msg":"f5245e69-5477-40fa-8e61-fea352b2811a active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.696463122Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3172025-01-14T16:17:15.132Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.696499513Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"client":"0","":"downstairs","session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3182025-01-14T16:17:15.132Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.696538303Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"client":"0","":"downstairs","session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3192025-01-14T16:17:15.132Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.696568424Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"client":"1","":"downstairs","session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3202025-01-14T16:17:15.132Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.696607194Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"client":"1","":"downstairs","session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3212025-01-14T16:17:15.132Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.696681765Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"client":"2","":"downstairs","session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3222025-01-14T16:17:15.132Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.696731886Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"client":"2","":"downstairs","session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3232025-01-14T16:17:15.132Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.696768736Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3242025-01-14T16:17:15.132Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.696803446Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3252025-01-14T16:17:15.132Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.696961848Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3262025-01-14T16:17:15.132Z{"msg":"ds_connection connected from Ok(127.0.0.1:33738)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.697034199Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3272025-01-14T16:17:15.132Z{"msg":"ds_connection connected from Ok(127.0.0.1:49896)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.69707072Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3282025-01-14T16:17:15.132Z{"msg":"ds_connection connected from Ok(127.0.0.1:49330)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.69710961Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3292025-01-14T16:17:15.132Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.697497285Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"client":"0","":"downstairs","session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3302025-01-14T16:17:15.133Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.697534405Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"client":"2","":"downstairs","session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3312025-01-14T16:17:15.133Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.697550615Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"client":"1","":"downstairs","session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3322025-01-14T16:17:15.133Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.698117952Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"client":"1","":"downstairs","session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3332025-01-14T16:17:15.133Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.698162353Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"client":"0","":"downstairs","session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3342025-01-14T16:17:15.133Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.698193583Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"client":"2","":"downstairs","session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3352025-01-14T16:17:15.133Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.698536777Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3362025-01-14T16:17:15.133Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.698575918Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3372025-01-14T16:17:15.133Z{"msg":"[0]R flush_numbers: [3999, 3999, 3999, 3999, 3999]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.698596268Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"downstairs","":"downstairs","session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3382025-01-14T16:17:15.133Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.698629399Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"downstairs","":"downstairs","session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3392025-01-14T16:17:15.133Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.698645569Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"downstairs","":"downstairs","session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3402025-01-14T16:17:15.133Z{"msg":"[1]R flush_numbers: [3999, 3999, 3999, 3999, 3999]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.698676639Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"downstairs","":"downstairs","session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3412025-01-14T16:17:15.133Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.698697789Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"downstairs","":"downstairs","session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3422025-01-14T16:17:15.133Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.69873332Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"downstairs","":"downstairs","session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3432025-01-14T16:17:15.133Z{"msg":"[2]R flush_numbers: [3999, 3999, 3999, 3999, 3999]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.69877016Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"downstairs","":"downstairs","session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3442025-01-14T16:17:15.133Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.69880327Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"downstairs","":"downstairs","session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3452025-01-14T16:17:15.133Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.698838511Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"downstairs","":"downstairs","session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3462025-01-14T16:17:15.133Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.698867461Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"downstairs","":"downstairs","session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3472025-01-14T16:17:15.133Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.698881862Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"downstairs","":"downstairs","session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3482025-01-14T16:17:15.133Z{"msg":"Next flush: 4000","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.698901322Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"downstairs","":"downstairs","session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3492025-01-14T16:17:15.133Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.698918222Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"downstairs","":"downstairs","session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3502025-01-14T16:17:15.133Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.698953192Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3512025-01-14T16:17:15.133Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.698980283Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"client":"0","":"downstairs","session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3522025-01-14T16:17:15.133Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.699019383Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"client":"1","":"downstairs","session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3532025-01-14T16:17:15.133Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.699052853Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"client":"2","":"downstairs","session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3542025-01-14T16:17:15.133Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.699080574Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3552025-01-14T16:17:15.133Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.699106854Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3562025-01-14T16:17:15.133Z{"msg":"f5245e69-5477-40fa-8e61-fea352b2811a is now active with session: eee38f73-1d48-47dd-95fa-4e7d6291c2f0","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.699252286Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3572025-01-14T16:17:15.133Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.699302347Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"session_id":"eee38f73-1d48-47dd-95fa-4e7d6291c2f0"}
3582025-01-14T16:17:15.133Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.699334017Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238}
3592025-01-14T16:17:15.133Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.699359577Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238}
3602025-01-14T16:17:15.133Z{"msg":"f5245e69-5477-40fa-8e61-fea352b2811a active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.699384148Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3612025-01-14T16:17:15.133Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.699416608Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"client":"0","":"downstairs","session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3622025-01-14T16:17:15.133Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.699433218Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"client":"0","":"downstairs","session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3632025-01-14T16:17:15.133Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.699446568Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"client":"1","":"downstairs","session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3642025-01-14T16:17:15.133Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.699468838Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"client":"1","":"downstairs","session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3652025-01-14T16:17:15.133Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.699482559Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"client":"2","":"downstairs","session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3662025-01-14T16:17:15.133Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.699497219Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"client":"2","":"downstairs","session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3672025-01-14T16:17:15.133Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.699517099Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3682025-01-14T16:17:15.134Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.699531679Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3692025-01-14T16:17:15.134Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.69954497Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3702025-01-14T16:17:15.134Z{"msg":"ds_connection connected from Ok(127.0.0.1:32960)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.6995666Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3712025-01-14T16:17:15.134Z{"msg":"ds_connection connected from Ok(127.0.0.1:43223)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.6995858Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3722025-01-14T16:17:15.134Z{"msg":"ds_connection connected from Ok(127.0.0.1:37733)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.699633251Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3732025-01-14T16:17:15.134Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.699665541Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"client":"1","":"downstairs","session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3742025-01-14T16:17:15.134Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.699710761Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"client":"0","":"downstairs","session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3752025-01-14T16:17:15.134Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.699745112Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"client":"2","":"downstairs","session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3762025-01-14T16:17:15.138Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.700130086Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"client":"1","":"downstairs","session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3772025-01-14T16:17:15.138Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.700173087Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"client":"0","":"downstairs","session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3782025-01-14T16:17:15.138Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.700198897Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"client":"2","":"downstairs","session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3792025-01-14T16:17:15.138Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.700482101Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3802025-01-14T16:17:15.138Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.700598222Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3812025-01-14T16:17:15.138Z{"msg":"[0]R flush_numbers: [5320, 5320, 5320, 5320, 5320]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.700685513Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"downstairs","":"downstairs","session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3822025-01-14T16:17:15.138Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.700719553Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"downstairs","":"downstairs","session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3832025-01-14T16:17:15.138Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.700748314Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"downstairs","":"downstairs","session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3842025-01-14T16:17:15.138Z{"msg":"[1]R flush_numbers: [5320, 5320, 5320, 5320, 5320]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.700773094Z","hostname":"Wait for a query_work_queue command to finish before sending IO
3852025-01-14T16:17:15.138Zw-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"downstairs","":"downstairs","session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3862025-01-14T16:17:15.138Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.700802834Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"downstairs","":"downstairs","session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3872025-01-14T16:17:15.138Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.700827425Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"downstairs","":"downstairs","session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3882025-01-14T16:17:15.138Z{"msg":"[2]R flush_numbers: [5320, 5320, 5320, 5320, 5320]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.700859695Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"downstairs","":"downstairs","session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3892025-01-14T16:17:15.138Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.700883366Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"downstairs","":"downstairs","session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3902025-01-14T16:17:15.138Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.700922596Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"downstairs","":"downstairs","session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3912025-01-14T16:17:15.138Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.700943466Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"downstairs","":"downstairs","session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3922025-01-14T16:17:15.139Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.700963567Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"downstairs","":"downstairs","session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3932025-01-14T16:17:15.139Z{"msg":"Next flush: 5321","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.700982317Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"downstairs","":"downstairs","session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3942025-01-14T16:17:15.139Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.701011317Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"":"downstairs","":"downstairs","session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3952025-01-14T16:17:15.139Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.701039297Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3962025-01-14T16:17:15.139Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.701056948Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"client":"0","":"downstairs","session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3972025-01-14T16:17:15.139Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.701086998Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"client":"1","":"downstairs","session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3982025-01-14T16:17:15.139Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.701101858Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"client":"2","":"downstairs","session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
3992025-01-14T16:17:15.139Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.701122508Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
4002025-01-14T16:17:15.139Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.701138769Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
4012025-01-14T16:17:15.139Z{"msg":"f5245e69-5477-40fa-8e61-fea352b2811a is now active with session: 2dac29f4-e938-4d00-aada-e4bdeb504a10","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.701170199Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
4022025-01-14T16:17:15.139Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.701190519Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238,"session_id":"2dac29f4-e938-4d00-aada-e4bdeb504a10"}
4032025-01-14T16:17:15.139Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:14.70121169Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238}
4042025-01-14T16:17:15.139ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
4052025-01-14T16:17:15.139ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4062025-01-14T16:17:15.139ZFill test
4072025-01-14T16:17:15.175ZRead and Verify all blocks (0..100 range:false)
4082025-01-14T16:17:15.253ZWrote out file "/var/tmp/test_up-build/verify_file"
4092025-01-14T16:17:15.253ZCLIENT: Tests done. All submitted work has been ACK'd
4102025-01-14T16:17:15.253Z----------------------------------------------------------------
4112025-01-14T16:17:15.253Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4122025-01-14T16:17:15.253ZStates: Active Active Active
4132025-01-14T16:17:15.253ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4142025-01-14T16:17:15.253Z 1002 Acked Read 50 Done Done Done false
4152025-01-14T16:17:15.253Z STATES DS:0 DS:1 DS:2 TOTAL
4162025-01-14T16:17:15.253Z Sent 0 0 0 0
4172025-01-14T16:17:15.253Z Done 1 1 1 3
4182025-01-14T16:17:15.253Z Skipped 0 0 0 0
4192025-01-14T16:17:15.253Z Error 0 0 0 0
4202025-01-14T16:17:15.256ZLast Flush: 1001 1001 1001
4212025-01-14T16:17:15.256ZDownstairs last five completed: 1001 1000
4222025-01-14T16:17:15.256ZUpstairs last five completed: 1002 1001 1000
4232025-01-14T16:17:15.256Z----------------------------------------------------------------
4242025-01-14T16:17:15.256Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4252025-01-14T16:17:15.256ZStates: Active Active Active
4262025-01-14T16:17:15.256ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4272025-01-14T16:17:15.256Z 1002 Acked Read 50 Sent Sent Done false
4282025-01-14T16:17:15.256Z STATES DS:0 DS:1 DS:2 TOTAL
4292025-01-14T16:17:15.256Z Sent 1 1 0 2
4302025-01-14T16:17:15.256Z Done 0 0 1 1
4312025-01-14T16:17:15.256Z Skipped 0 0 0 0
4322025-01-14T16:17:15.256Z Error 0 0 0 0
4332025-01-14T16:17:15.256ZLast Flush: 1001 1001 1001
4342025-01-14T16:17:15.256ZDownstairs last five completed: 1001 1000
4352025-01-14T16:17:15.256ZUpstairs last five completed: 1002 1001 1000
4362025-01-14T16:17:15.256ZCLIENT: Up:0 ds:2 act:6
4372025-01-14T16:17:19.255Z----------------------------------------------------------------
4382025-01-14T16:17:19.255Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4392025-01-14T16:17:19.255ZDownstairs last five completed: 1003 1002 1001 1000
4402025-01-14T16:17:19.255ZUpstairs last five completed: 1003 1002 1001 1000
4412025-01-14T16:17:19.255Z----------------------------------------------------------------
4422025-01-14T16:17:19.255Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4432025-01-14T16:17:19.255ZDownstairs last five completed: 1003 1002 1001 1000
4442025-01-14T16:17:19.255ZUpstairs last five completed: 1003 1002 1001 1000
4452025-01-14T16:17:19.258ZCLIENT: Up:0 ds:0 act:6
4462025-01-14T16:17:19.258ZCLIENT: All crucible jobs finished, exiting program
4472025-01-14T16:17:19.258Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:18.823492882Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1238}
4482025-01-14T16:17:19.262ZRepair part 1 passed
4492025-01-14T16:17:19.264Z
4502025-01-14T16:17:19.264ZKill the current downstairs
4512025-01-14T16:17:19.428ZDownstairs 2 stopped
4522025-01-14T16:17:19.428Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
4532025-01-14T16:17:19.431ZNow put back the original so we have a mismatch
4542025-01-14T16:17:19.431Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
4552025-01-14T16:17:19.433ZRestart downstairs with old directory
4562025-01-14T16:17:19.485Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4572025-01-14T16:17:19.501ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4582025-01-14T16:17:19.504Z 0 000-009 32 32 31 4000 4000 3999 F F F <---
4592025-01-14T16:17:19.504Z 1 010-019 32 32 31 4000 4000 3999 F F F <---
4602025-01-14T16:17:19.504Z 2 020-029 32 32 31 4000 4000 3999 F F F <---
4612025-01-14T16:17:19.504Z 3 030-039 32 32 31 4000 4000 3999 F F F <---
4622025-01-14T16:17:19.504Z 4 040-049 32 32 31 4000 4000 3999 F F F <---
4632025-01-14T16:17:19.504ZMax gen: 32, Max flush: 4000
4642025-01-14T16:17:19.504ZError: Difference in extent metadata found!
4652025-01-14T16:17:19.506Zdump test found error as expected
4662025-01-14T16:17:19.508Z
4672025-01-14T16:17:19.508Z
4682025-01-14T16:17:19.508Z/var/tmp/bins/crutest verify --range -g 33 -q --key 93Moh3MNIMkYzI46QoVAVPKhai1YPtEopcGQw+wY/3w= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
4692025-01-14T16:17:19.561Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.129356613Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"task":"crutest"}
4702025-01-14T16:17:19.564Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.129846109Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"task":"crutest"}
4712025-01-14T16:17:19.564Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.131494878Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"task":"crutest"}
4722025-01-14T16:17:19.564Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.1316796Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246}
4732025-01-14T16:17:19.564Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.131724351Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246}
4742025-01-14T16:17:19.564Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.131746921Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246}
4752025-01-14T16:17:19.564Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.131761941Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246}
4762025-01-14T16:17:19.564Z{"msg":"Crucible 3df1422f-4277-4018-8c77-3646c40bad7f has session id: 68a523dc-d759-4df8-8824-2ff49fb5a016","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.1324299Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
4772025-01-14T16:17:19.564Z{"msg":"Upstairs opts: Upstairs UUID: 3df1422f-4277-4018-8c77-3646c40bad7f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.13248342Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
4782025-01-14T16:17:19.568Z{"msg":"Crucible stats registered with UUID: 3df1422f-4277-4018-8c77-3646c40bad7f","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.132507001Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
4792025-01-14T16:17:19.568Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:19.132542811Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
4802025-01-14T16:17:19.568Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.134447764Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"task":"crutest"}
4812025-01-14T16:17:19.568Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.134538465Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246}
4822025-01-14T16:17:19.568Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.134592575Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246}
4832025-01-14T16:17:19.568Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.134624836Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246}
4842025-01-14T16:17:19.568Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.134669476Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246}
4852025-01-14T16:17:19.568Z{"msg":"Crucible 3df1422f-4277-4018-8c77-3646c40bad7f has session id: e9fd3246-822f-4a28-beed-3f84dca1e45f","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.1349972Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
4862025-01-14T16:17:19.568Z{"msg":"Upstairs opts: Upstairs UUID: 3df1422f-4277-4018-8c77-3646c40bad7f, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.135045691Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
4872025-01-14T16:17:19.569Z{"msg":"Crucible stats registered with UUID: 3df1422f-4277-4018-8c77-3646c40bad7f","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.135069401Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
4882025-01-14T16:17:19.569Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:19.135106742Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
4892025-01-14T16:17:19.569Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.13584735Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"task":"crutest"}
4902025-01-14T16:17:19.569Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.135891431Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246}
4912025-01-14T16:17:19.569Z{"msg":"3df1422f-4277-4018-8c77-3646c40bad7f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.135989342Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
4922025-01-14T16:17:19.569Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.136030203Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"0","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
4932025-01-14T16:17:19.569Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.136071633Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"0","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
4942025-01-14T16:17:19.569Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.136107514Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"1","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
4952025-01-14T16:17:19.569Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.136132564Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"1","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
4962025-01-14T16:17:19.569Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.136165004Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"2","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
4972025-01-14T16:17:19.571Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.136190695Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"2","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
4982025-01-14T16:17:19.571Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.136227345Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
4992025-01-14T16:17:19.571Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.136253226Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5002025-01-14T16:17:19.571Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.136286726Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5012025-01-14T16:17:19.571Z{"msg":"ds_connection connected from Ok(127.0.0.1:45619)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.136311576Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5022025-01-14T16:17:19.571Z{"msg":"ds_connection connected from Ok(127.0.0.1:55298)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.136337396Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5032025-01-14T16:17:19.571Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:19.136366537Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5042025-01-14T16:17:19.574Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:19.136441798Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5052025-01-14T16:17:19.574Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:19.136485208Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5062025-01-14T16:17:19.574Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.136505109Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5072025-01-14T16:17:19.574Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.136892943Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"0","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5082025-01-14T16:17:19.574Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.136938384Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"1","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5092025-01-14T16:17:19.574Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.1374709Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"0","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5102025-01-14T16:17:19.574Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.137558111Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"1","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5112025-01-14T16:17:19.574Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.137840774Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5122025-01-14T16:17:19.574Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.137920426Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5132025-01-14T16:17:29.569Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.137555081Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5142025-01-14T16:17:29.569Z{"msg":"ds_connection connected from Ok(127.0.0.1:42259)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.137624542Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5152025-01-14T16:17:29.573Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.138813236Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"2","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5162025-01-14T16:17:29.573Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.139461014Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"2","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5172025-01-14T16:17:29.573Z{"msg":"[0]R flush_numbers: [4000, 4000, 4000, 4000, 4000]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.139868179Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5182025-01-14T16:17:29.573Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.139906309Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5192025-01-14T16:17:29.573Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.139923789Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5202025-01-14T16:17:29.573Z{"msg":"[1]R flush_numbers: [4000, 4000, 4000, 4000, 4000]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.13993779Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5212025-01-14T16:17:29.573Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.139951Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5222025-01-14T16:17:29.573Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.1399653Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5232025-01-14T16:17:29.573Z{"msg":"[2]R flush_numbers: [3999, 3999, 3999, 3999, 3999]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.13997864Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5242025-01-14T16:17:29.573Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.13999943Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5252025-01-14T16:17:29.573Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140013341Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5262025-01-14T16:17:29.573Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140026331Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5272025-01-14T16:17:29.573Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140046751Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5282025-01-14T16:17:29.573Z{"msg":"Next flush: 4001","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140060061Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5292025-01-14T16:17:29.573Z{"msg":"Extent 0 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140073151Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5302025-01-14T16:17:29.573Z{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140093711Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5312025-01-14T16:17:29.573Z{"msg":"extent:0 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140108142Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5322025-01-14T16:17:29.573Z{"msg":"extent:0 flush: 4000 4000 3999 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140123282Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5332025-01-14T16:17:29.573Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140150372Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5342025-01-14T16:17:29.573Z{"msg":"extent:0 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140164602Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5352025-01-14T16:17:29.573Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140191533Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5362025-01-14T16:17:29.573Z{"msg":"find dest for source 0 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140212913Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5372025-01-14T16:17:29.573Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140235833Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5382025-01-14T16:17:29.573Z{"msg":"Extent 1 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140257204Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5392025-01-14T16:17:29.573Z{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140278884Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5402025-01-14T16:17:29.573Z{"msg":"extent:1 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140293164Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5412025-01-14T16:17:29.573Z{"msg":"extent:1 flush: 4000 4000 3999 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140314114Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5422025-01-14T16:17:29.573Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140328664Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5432025-01-14T16:17:29.573Z{"msg":"extent:1 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140342805Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5442025-01-14T16:17:29.574Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140363745Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5452025-01-14T16:17:29.574Z{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140378365Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5462025-01-14T16:17:29.574Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140392195Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5472025-01-14T16:17:29.577Z{"msg":"Extent 2 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140406465Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5482025-01-14T16:17:29.577Z{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140461566Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5492025-01-14T16:17:29.578Z{"msg":"extent:2 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140479586Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5502025-01-14T16:17:29.578Z{"msg":"extent:2 flush: 4000 4000 3999 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140493986Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5512025-01-14T16:17:29.578Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140515987Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5522025-01-14T16:17:29.578Z{"msg":"extent:2 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140530437Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5532025-01-14T16:17:29.578Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140551797Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5542025-01-14T16:17:29.578Z{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140566217Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5552025-01-14T16:17:29.578Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140597878Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5562025-01-14T16:17:29.578Z{"msg":"Extent 3 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140623508Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5572025-01-14T16:17:29.578Z{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140637828Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5582025-01-14T16:17:29.578Z{"msg":"extent:3 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140664468Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5592025-01-14T16:17:29.578Z{"msg":"extent:3 flush: 4000 4000 3999 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140697939Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5602025-01-14T16:17:29.578Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140730759Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5612025-01-14T16:17:29.578Z{"msg":"extent:3 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.14076482Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5622025-01-14T16:17:29.579Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.14080038Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5632025-01-14T16:17:29.579Z{"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.14083308Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5642025-01-14T16:17:29.579Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140868381Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5652025-01-14T16:17:29.579Z{"msg":"Extent 4 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140891201Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5662025-01-14T16:17:29.579Z{"msg":"First source client ID for extent 4","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140928232Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5672025-01-14T16:17:29.579Z{"msg":"extent:4 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140944162Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5682025-01-14T16:17:29.579Z{"msg":"extent:4 flush: 4000 4000 3999 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140961702Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5692025-01-14T16:17:29.579Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.140997342Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5702025-01-14T16:17:29.579Z{"msg":"extent:4 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.141014403Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5712025-01-14T16:17:29.579Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.141028403Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5722025-01-14T16:17:29.579Z{"msg":"find dest for source 0 for extent at index 4","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.141049693Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5732025-01-14T16:17:29.579Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.141063753Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5742025-01-14T16:17:29.579Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.141117114Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"0","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5752025-01-14T16:17:29.579Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.141145594Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"1","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5762025-01-14T16:17:29.579Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.141159454Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"2","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5772025-01-14T16:17:29.579Z{"msg":"Full repair list: {ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.141183995Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5782025-01-14T16:17:29.580Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 4001, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 4001, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 4001, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 4001, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 4001, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.141271556Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5792025-01-14T16:17:29.580Z{"msg":"starting reconciliation 63e3a654-5207-4e15-a389-f675d18437d0: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.141295506Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5802025-01-14T16:17:29.580Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.141309096Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"2","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5812025-01-14T16:17:29.583Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.149307602Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"2","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5822025-01-14T16:17:29.586Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.152907395Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"2","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5832025-01-14T16:17:29.588Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.156318846Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"2","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5842025-01-14T16:17:29.591Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.159877109Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"2","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5852025-01-14T16:17:29.594Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.162877895Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"downstairs","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5862025-01-14T16:17:29.596Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.162913555Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"0","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5872025-01-14T16:17:29.596Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.162929485Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"1","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5882025-01-14T16:17:29.596Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.162943566Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"2","":"downstairs","session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5892025-01-14T16:17:29.596Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.162964386Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5902025-01-14T16:17:29.599Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.162977026Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5912025-01-14T16:17:29.599Z{"msg":"3df1422f-4277-4018-8c77-3646c40bad7f is now active with session: 68a523dc-d759-4df8-8824-2ff49fb5a016","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.162997256Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"session_id":"68a523dc-d759-4df8-8824-2ff49fb5a016"}
5922025-01-14T16:17:29.599Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.163019487Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246}
5932025-01-14T16:17:29.599Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.163044947Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246}
5942025-01-14T16:17:29.599Z{"msg":"3df1422f-4277-4018-8c77-3646c40bad7f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.163066217Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
5952025-01-14T16:17:29.599Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.163087407Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"0","":"downstairs","session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
5962025-01-14T16:17:29.599Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.163101898Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"0","":"downstairs","session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
5972025-01-14T16:17:29.599Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.163114738Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"1","":"downstairs","session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
5982025-01-14T16:17:29.599Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.163135218Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"1","":"downstairs","session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
5992025-01-14T16:17:29.599Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.163148288Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"2","":"downstairs","session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6002025-01-14T16:17:29.599Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.163165048Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"2","":"downstairs","session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6012025-01-14T16:17:29.599Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.163198529Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6022025-01-14T16:17:29.599Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.163223829Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6032025-01-14T16:17:29.600Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.16325934Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6042025-01-14T16:17:29.600Z{"msg":"ds_connection connected from Ok(127.0.0.1:57193)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.1633003Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6052025-01-14T16:17:29.600Z{"msg":"ds_connection connected from Ok(127.0.0.1:46385)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.16332883Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6062025-01-14T16:17:29.600Z{"msg":"ds_connection connected from Ok(127.0.0.1:63812)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.163357051Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6072025-01-14T16:17:29.600Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.163697785Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"1","":"downstairs","session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6082025-01-14T16:17:29.600Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.163752026Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"2","":"downstairs","session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6092025-01-14T16:17:29.600Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.163785296Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"0","":"downstairs","session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6102025-01-14T16:17:29.600Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.164284682Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"1","":"downstairs","session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6112025-01-14T16:17:29.600Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.164408893Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"0","":"downstairs","session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6122025-01-14T16:17:29.600Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.164453694Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"2","":"downstairs","session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6132025-01-14T16:17:29.600Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.164691757Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6142025-01-14T16:17:29.600Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.164727837Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6152025-01-14T16:17:29.600Z{"msg":"[0]R flush_numbers: [5321, 5321, 5321, 5321, 5321]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.164818828Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"downstairs","":"downstairs","session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6162025-01-14T16:17:29.600Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.164856459Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"downstairs","":"downstairs","session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6172025-01-14T16:17:29.600Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.164873369Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"downstairs","":"downstairs","session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6182025-01-14T16:17:29.600Z{"msg":"[1]R flush_numbers: [5321, 5321, 5321, 5321, 5321]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.164887849Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"downstairs","":"downstairs","session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6192025-01-14T16:17:29.600Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.164905939Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"downstairs","":"downstairs","session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6202025-01-14T16:17:29.600Z{"msg":"Wait for a query_work_queue command to finish before sending IO
6212025-01-14T16:17:29.600Z[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.16494733Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"downstairs","":"downstairs","session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6222025-01-14T16:17:29.600Z{"msg":"[2]R flush_numbers: [5321, 5321, 5321, 5321, 5321]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.16497279Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"downstairs","":"downstairs","session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6232025-01-14T16:17:29.600Z{"msg":"[2]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.16498699Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"downstairs","":"downstairs","session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6242025-01-14T16:17:29.600Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.16500933Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"downstairs","":"downstairs","session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6252025-01-14T16:17:29.600Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.165045251Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"downstairs","":"downstairs","session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6262025-01-14T16:17:29.600Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.165082301Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"downstairs","":"downstairs","session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6272025-01-14T16:17:29.600Z{"msg":"Next flush: 5322","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.165102062Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"downstairs","":"downstairs","session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6282025-01-14T16:17:29.600Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.165135182Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"":"downstairs","":"downstairs","session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6292025-01-14T16:17:29.601Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.165164572Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6302025-01-14T16:17:29.601Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.165187263Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"0","":"downstairs","session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6312025-01-14T16:17:29.601Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.165219663Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"1","":"downstairs","session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6322025-01-14T16:17:29.601Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.165241473Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"client":"2","":"downstairs","session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6332025-01-14T16:17:29.601Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.165284964Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6342025-01-14T16:17:29.601Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.165303164Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6352025-01-14T16:17:29.601Z{"msg":"3df1422f-4277-4018-8c77-3646c40bad7f is now active with session: e9fd3246-822f-4a28-beed-3f84dca1e45f","v":0,"name":"crucible","level":30Disk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
6362025-01-14T16:17:29.601Z,"time":"2025-01-14T16:17:29.165335394Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6372025-01-14T16:17:29.601Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.165357195Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246,"session_id":"e9fd3246-822f-4a28-beed-3f84dca1e45f"}
6382025-01-14T16:17:29.601Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:29.165379155Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246}
6392025-01-14T16:17:29.601ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
6402025-01-14T16:17:29.601ZRead and Verify all blocks (0..100 range:true)
6412025-01-14T16:17:29.679ZWrote out file "/var/tmp/test_up-build/verify_file"
6422025-01-14T16:17:29.679ZVerify test completed
6432025-01-14T16:17:29.681ZWrote out file "/var/tmp/test_up-build/verify_file"
6442025-01-14T16:17:29.681ZCLIENT: Tests done. All submitted work has been ACK'd
6452025-01-14T16:17:29.682Z----------------------------------------------------------------
6462025-01-14T16:17:29.682Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6472025-01-14T16:17:29.682ZStates: Active Active Active
6482025-01-14T16:17:29.682ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6492025-01-14T16:17:29.682Z 1000 Acked Read 50 Done Done Done false
6502025-01-14T16:17:29.682Z STATES DS:0 DS:1 DS:2 TOTAL
6512025-01-14T16:17:29.683Z Sent 0 0 0 0
6522025-01-14T16:17:29.683Z Done 1 1 1 3
6532025-01-14T16:17:29.683Z Skipped 0 0 0 0
6542025-01-14T16:17:29.683Z Error 0 0 0 0
6552025-01-14T16:17:29.683ZLast Flush: 0 0 0
6562025-01-14T16:17:29.683ZDownstairs last five completed:
6572025-01-14T16:17:29.683ZUpstairs last five completed: 1000
6582025-01-14T16:17:29.683Z----------------------------------------------------------------
6592025-01-14T16:17:29.683Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6602025-01-14T16:17:29.683ZStates: Active Active Active
6612025-01-14T16:17:29.683ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6622025-01-14T16:17:29.683Z 1000 Acked Read 50 Sent Done Sent false
6632025-01-14T16:17:29.683Z STATES DS:0 DS:1 DS:2 TOTAL
6642025-01-14T16:17:29.683Z Sent 1 0 1 2
6652025-01-14T16:17:29.683Z Done 0 1 0 1
6662025-01-14T16:17:29.683Z Skipped 0 0 0 0
6672025-01-14T16:17:29.683Z Error 0 0 0 0
6682025-01-14T16:17:29.683ZLast Flush: 0 0 0
6692025-01-14T16:17:29.683ZDownstairs last five completed:
6702025-01-14T16:17:29.683ZUpstairs last five completed: 1000
6712025-01-14T16:17:29.683ZCLIENT: Up:0 ds:2 act:6
6722025-01-14T16:17:33.681Z----------------------------------------------------------------
6732025-01-14T16:17:33.681Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6742025-01-14T16:17:33.681ZDownstairs last five completed: 1001 1000
6752025-01-14T16:17:33.681ZUpstairs last five completed: 1001 1000
6762025-01-14T16:17:33.684Z----------------------------------------------------------------
6772025-01-14T16:17:33.685Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6782025-01-14T16:17:33.685ZDownstairs last five completed: 1001 1000
6792025-01-14T16:17:33.685ZUpstairs last five completed: 1001 1000
6802025-01-14T16:17:33.685ZCLIENT: Up:0 ds:0 act:6
6812025-01-14T16:17:33.685ZCLIENT: All crucible jobs finished, exiting program
6822025-01-14T16:17:33.685Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:33.249515371Z","hostname":"w-01JHJS779VQF5PA1M2YNG80QCD","pid":1246}
6832025-01-14T16:17:33.688ZRepair part 2 passed
6842025-01-14T16:17:33.690Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
6852025-01-14T16:17:33.705ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6862025-01-14T16:17:33.705Z 0 000-009 32 32 32 4000 4000 4000 F F F
6872025-01-14T16:17:33.705Z 1 010-019 32 32 32 4000 4000 4000 F F F
6882025-01-14T16:17:33.705Z 2 020-029 32 32 32 4000 4000 4000 F F F
6892025-01-14T16:17:33.708Z 3 030-039 32 32 32 4000 4000 4000 F F F
6902025-01-14T16:17:33.708Z 4 040-049 32 32 32 4000 4000 4000 F F F
6912025-01-14T16:17:33.708ZMax gen: 32, Max flush: 4000
6922025-01-14T16:17:33.708Zdump test passed
6932025-01-14T16:17:33.708Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
6942025-01-14T16:17:33.722Z Extent 1
6952025-01-14T16:17:33.725ZGEN 32 32 32
6962025-01-14T16:17:33.725ZFLUSH_ID 4000 4000 4000
6972025-01-14T16:17:33.725ZDIRTY
6982025-01-14T16:17:33.725Z
6992025-01-14T16:17:33.725ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
7002025-01-14T16:17:33.725Z010 A A A A A A
7012025-01-14T16:17:33.728Z011 A A A A A A
7022025-01-14T16:17:33.728Z012 A A A A A A
7032025-01-14T16:17:33.731Z013 A A A A A A
7042025-01-14T16:17:33.731Z014 A A A A A A
7052025-01-14T16:17:33.733Z015 A A A A A A
7062025-01-14T16:17:33.733Z016 A A A A A A
7072025-01-14T16:17:33.736Z017 A A A A A A
7082025-01-14T16:17:33.736Z018 A A A A A A
7092025-01-14T16:17:33.739Z019 A A A A A A
7102025-01-14T16:17:33.742Zdump extent test passed
7112025-01-14T16:17:33.742Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
7122025-01-14T16:17:33.754ZExtent 2 Block in extent 0 Actual block 20
7132025-01-14T16:17:33.754Z
7142025-01-14T16:17:33.757Z DATA SHA256 VER
7152025-01-14T16:17:33.757Z------ ---------------------------------------------------------------- ---
7162025-01-14T16:17:33.757Z 0 25ee0499f9ed4734803eed94728e1eb1f96c4f89f0f79aca0117eec76a17512c A
7172025-01-14T16:17:33.761Z 1 25ee0499f9ed4734803eed94728e1eb1f96c4f89f0f79aca0117eec76a17512c A
7182025-01-14T16:17:33.761Z 2 25ee0499f9ed4734803eed94728e1eb1f96c4f89f0f79aca0117eec76a17512c A
7192025-01-14T16:17:33.761Z
7202025-01-14T16:17:33.761ZNONCES 0 1 2 DIFF
7212025-01-14T16:17:33.761Z------ ------------------------ ------------------------ ------------------------ -----
7222025-01-14T16:17:33.761Z 0 aeb1f8a7955351e4c77186a9 aeb1f8a7955351e4c77186a9 aeb1f8a7955351e4c77186a9
7232025-01-14T16:17:33.761Z
7242025-01-14T16:17:33.761Z TAGS 0 1 2 DIFF
7252025-01-14T16:17:33.761Z------ -------------------------------- -------------------------------- -------------------------------- -----
7262025-01-14T16:17:33.761Z 0 396fa828e31127bba7c2fd62df400161 396fa828e31127bba7c2fd62df400161 396fa828e31127bba7c2fd62df400161
7272025-01-14T16:17:33.761Z
7282025-01-14T16:17:33.761ZHASHES 0 1 2 DIFF
7292025-01-14T16:17:33.761Z------ ---------------- ---------------- ---------------- -----
7302025-01-14T16:17:33.761Z
7312025-01-14T16:17:33.766Zdump block test passed
7322025-01-14T16:17:33.766ZInitial upstairs tests have completed, stopping all downstairs
7332025-01-14T16:17:38.824ZCreating 7 larger downstairs regions
7342025-01-14T16:17:39.000ZStarting 7 downstairs
7352025-01-14T16:17:44.006Zdsc restarted at PID: 1262
7362025-01-14T16:17:44.378ZNow do the replace-reconcile test
7372025-01-14T16:17:44.431ZUsing 8870 for the replacement port
7382025-01-14T16:20:39.296ZCompleted test: replace-reconcile
7392025-01-14T16:20:39.299ZNow do the replace-before-active test
7402025-01-14T16:24:12.979ZCompleted test: replace-before-active
7412025-01-14T16:24:12.979ZAll tests have completed, stopping all downstairs
7422025-01-14T16:24:13.038Z
7432025-01-14T16:24:13.038ZAll Tests have passed
7442025-01-14T16:24:13.040Z13:39 Test duration
7452025-01-14T16:24:13.040Z
7462025-01-14T16:24:13.040Zreal 13:39.512412140
7472025-01-14T16:24:13.041Zuser 18:27.390852055
7482025-01-14T16:24:13.041Zsys 23.748560146
7492025-01-14T16:24:13.041Ztrap 0.532028454
7502025-01-14T16:24:13.041Ztflt 0.038003167
7512025-01-14T16:24:13.041Zdflt 0.039134964
7522025-01-14T16:24:13.041Zkflt 0.001438485
7532025-01-14T16:24:13.041Zlock 25:38:29.534733372
7542025-01-14T16:24:13.041Zslp 1:14:54.317050166
7552025-01-14T16:24:13.041Zlat 1:59.934737933
7562025-01-14T16:24:13.041Zstop 0.333265562
7572025-01-14T16:24:13.041Z+ echo 'test-up-2region-encrypted ends'
7582025-01-14T16:24:13.041Ztest-up-2region-encrypted ends
7592025-01-14T16:24:18.046Zprocess exited: duration 827691 ms, exit code 0
7602025-01-14T16:24:18.046Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
7612025-01-14T16:25:18.048Zexec warning: : stdout descriptor may be held open by a background process; giving up!
7622025-01-14T16:25:18.048Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
7632025-01-14T16:25:18.057Zfound 17 output files
7642025-01-14T16:25:18.065Zuploading: /tmp/test_up-build/dsc-out.txt (50896 bytes)
7652025-01-14T16:25:19.072Zuploaded: /tmp/test_up-build/dsc-out.txt
7662025-01-14T16:25:19.072Zuploading: /tmp/test_up-build/test_up_out.txt (10384064 bytes)
7672025-01-14T16:25:20.251Zuploaded: /tmp/test_up-build/test_up_out.txt
7682025-01-14T16:25:20.251Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (147257 bytes)
7692025-01-14T16:25:21.271Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
7702025-01-14T16:25:21.271Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (144661 bytes)
7712025-01-14T16:25:22.289Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
7722025-01-14T16:25:22.289Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (250968 bytes)
7732025-01-14T16:25:23.313Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
7742025-01-14T16:25:23.313Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (139424 bytes)
7752025-01-14T16:25:24.330Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
7762025-01-14T16:25:24.330Zuploading: /tmp/test_up-build/dsc/downstairs-8850.txt (2504 bytes)
7772025-01-14T16:25:25.341Zuploaded: /tmp/test_up-build/dsc/downstairs-8850.txt
7782025-01-14T16:25:25.342Zuploading: /tmp/test_up-build/dsc/downstairs-8860.txt (33796 bytes)
7792025-01-14T16:25:26.352Zuploaded: /tmp/test_up-build/dsc/downstairs-8860.txt
7802025-01-14T16:25:26.355Zuploading: /tmp/test_up-build/dsc/downstairs-8870.txt (738191 bytes)
7812025-01-14T16:25:27.391Zuploaded: /tmp/test_up-build/dsc/downstairs-8870.txt
7822025-01-14T16:25:27.391Zuploading: /tmp/debug/df.txt (1270 bytes)
7832025-01-14T16:25:28.399Zuploaded: /tmp/debug/df.txt
7842025-01-14T16:25:28.399Zuploading: /tmp/debug/dtrace.txt (992423 bytes)
7852025-01-14T16:25:29.443Zuploaded: /tmp/debug/dtrace.txt
7862025-01-14T16:25:29.443Zuploading: /tmp/debug/iostat.txt (265562 bytes)
7872025-01-14T16:25:29.463Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 265562 -> 268862
7882025-01-14T16:25:30.470Zuploaded: /tmp/debug/iostat.txt
7892025-01-14T16:25:30.470Zuploading: /tmp/debug/mpstat.txt (645174 bytes)
7902025-01-14T16:25:30.502Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 645174 -> 653922
7912025-01-14T16:25:31.509Zuploaded: /tmp/debug/mpstat.txt
7922025-01-14T16:25:31.509Zuploading: /tmp/debug/paging.txt (115837 bytes)
7932025-01-14T16:25:31.521Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 115837 -> 117490
7942025-01-14T16:25:32.528Zuploaded: /tmp/debug/paging.txt
7952025-01-14T16:25:32.528Zuploading: /tmp/debug/prstat.txt (1253690 bytes)
7962025-01-14T16:25:32.577Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 1253690 -> 1271804
7972025-01-14T16:25:33.583Zuploaded: /tmp/debug/prstat.txt
7982025-01-14T16:25:33.583Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
7992025-01-14T16:25:34.591Zuploaded: /tmp/debug/psrinfo.txt
8002025-01-14T16:25:34.591Zuploading: /tmp/debug/upstairs-info.txt (384127 bytes)
8012025-01-14T16:25:35.622Zuploaded: /tmp/debug/upstairs-info.txt