01JJQDM3D8VSNZD1RA5X7PZ310: test-up-2region-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01JJQDMF7YMMSMVEKM45ZTSMRP

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12025-01-28T21:52:54.072Zjob dependencies complete; ready to run (waiting for 14 m 8 s)
22025-01-28T21:53:58.347Zjob assigned to worker 01JJQEEPB67WVEEERWJF95B088 [factory aws, i-0ce6f0f3c176beb5d] (queued for 1 m 4 s)
32025-01-28T21:54:05.911Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42025-01-28T21:54:07.081Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52025-01-28T21:54:07.081Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62025-01-28T21:54:08.170Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72025-01-28T21:54:08.170Zdownloading input: /input/build/work/bins/crutest.gz
82025-01-28T21:54:09.582Zdownloaded input: /input/build/work/bins/crutest.gz
92025-01-28T21:54:09.582Zdownloading input: /input/build/work/bins/dsc.gz
102025-01-28T21:54:10.349Zdownloaded input: /input/build/work/bins/dsc.gz
112025-01-28T21:54:10.353Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122025-01-28T21:54:10.367Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132025-01-28T21:54:10.367Zdownloading input: /input/build/work/scripts/test_ds.sh
142025-01-28T21:54:10.386Zdownloaded input: /input/build/work/scripts/test_ds.sh
152025-01-28T21:54:10.386Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162025-01-28T21:54:10.423Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172025-01-28T21:54:10.423Zdownloading input: /input/build/work/scripts/test_repair.sh
182025-01-28T21:54:10.443Zdownloaded input: /input/build/work/scripts/test_repair.sh
192025-01-28T21:54:10.443Zdownloading input: /input/build/work/scripts/test_replay.sh
202025-01-28T21:54:10.461Zdownloaded input: /input/build/work/scripts/test_replay.sh
212025-01-28T21:54:10.461Zdownloading input: /input/build/work/scripts/test_up.sh
222025-01-28T21:54:10.482Zdownloaded input: /input/build/work/scripts/test_up.sh
232025-01-28T21:54:10.482Zdownloading input: /input/build/work/scripts/upstairs_info.d
242025-01-28T21:54:10.499Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252025-01-28T21:54:10.502Zdownloading input: /input/build/tmp/cargo-test-out.log
262025-01-28T21:54:11.159Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272025-01-28T21:54:11.159Zstarting task 0: "setup"
282025-01-28T21:54:11.170Z++ uname -s
292025-01-28T21:54:11.176Z+ kern=SunOS
302025-01-28T21:54:11.176Z+ case "$kern" in
312025-01-28T21:54:11.176Z+ groupadd -g 12345 build
322025-01-28T21:54:11.181Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332025-01-28T21:54:13.182Z+ zfs create -o mountpoint=/work rpool/work
342025-01-28T21:54:13.385Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352025-01-28T21:54:13.388Z+ home_fs=zfs
362025-01-28T21:54:13.388Z+ [[ zfs == autofs ]]
372025-01-28T21:54:13.388Z+ mkdir -p /home/build
382025-01-28T21:54:13.391Z+ chown build:build /home/build /work
392025-01-28T21:54:15.392Z+ chmod 0700 /home/build /work
402025-01-28T21:54:15.395Zprocess exited: duration 4235 ms, exit code 0
 
412025-01-28T21:54:15.401Zstarting task 1: "authentication"
422025-01-28T21:54:15.418Zprocess exited: duration 16 ms, exit code 0
 
432025-01-28T21:54:15.423Zstarting task 2: "build"
442025-01-28T21:54:15.428Z+ banner cores
452025-01-28T21:54:15.432Z
462025-01-28T21:54:15.432Z #### #### ##### ###### ####
472025-01-28T21:54:15.432Z # # # # # # # #
482025-01-28T21:54:15.432Z # # # # # ##### ####
492025-01-28T21:54:15.432Z # # # ##### # #
502025-01-28T21:54:15.432Z # # # # # # # # #
512025-01-28T21:54:15.432Z #### #### # # ###### ####
522025-01-28T21:54:15.432Z
532025-01-28T21:54:15.435Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542025-01-28T21:54:15.439Z+ banner unpack
552025-01-28T21:54:15.443Z
562025-01-28T21:54:15.443Z # # # # ##### ## #### # #
572025-01-28T21:54:15.443Z # # ## # # # # # # # # #
582025-01-28T21:54:15.443Z # # # # # # # # # # ####
592025-01-28T21:54:15.443Z # # # # # ##### ###### # # #
602025-01-28T21:54:15.443Z # # # ## # # # # # # #
612025-01-28T21:54:15.443Z #### # # # # # #### # #
622025-01-28T21:54:15.443Z
632025-01-28T21:54:15.443Z+ mkdir -p /var/tmp/bins
642025-01-28T21:54:15.445Z+ for t in "$input/bins/"*.gz
652025-01-28T21:54:15.445Z++ basename /input/build/work/bins/crucible-downstairs.gz
662025-01-28T21:54:15.445Z+ b=crucible-downstairs.gz
672025-01-28T21:54:15.445Z+ b=crucible-downstairs
682025-01-28T21:54:15.445Z+ gunzip
692025-01-28T21:54:17.833Z+ chmod +x /var/tmp/bins/crucible-downstairs
702025-01-28T21:54:17.835Z+ for t in "$input/bins/"*.gz
712025-01-28T21:54:17.840Z++ basename /input/build/work/bins/crucible-hammer.gz
722025-01-28T21:54:17.840Z+ b=crucible-hammer.gz
732025-01-28T21:54:17.840Z+ b=crucible-hammer
742025-01-28T21:54:17.840Z+ gunzip
752025-01-28T21:54:19.978Z+ chmod +x /var/tmp/bins/crucible-hammer
762025-01-28T21:54:19.981Z+ for t in "$input/bins/"*.gz
772025-01-28T21:54:19.983Z++ basename /input/build/work/bins/crutest.gz
782025-01-28T21:54:19.983Z+ b=crutest.gz
792025-01-28T21:54:19.983Z+ b=crutest
802025-01-28T21:54:19.983Z+ gunzip
812025-01-28T21:54:22.437Z+ chmod +x /var/tmp/bins/crutest
822025-01-28T21:54:22.440Z+ for t in "$input/bins/"*.gz
832025-01-28T21:54:22.443Z++ basename /input/build/work/bins/dsc.gz
842025-01-28T21:54:22.443Z+ b=dsc.gz
852025-01-28T21:54:22.443Z+ b=dsc
862025-01-28T21:54:22.443Z+ gunzip
872025-01-28T21:54:23.778Z+ chmod +x /var/tmp/bins/dsc
882025-01-28T21:54:23.780Z+ export BINDIR=/var/tmp/bins
892025-01-28T21:54:23.780Z+ BINDIR=/var/tmp/bins
902025-01-28T21:54:23.780Z+ jobpid=1139
912025-01-28T21:54:23.783Z+ echo 'Setup debug logging'
922025-01-28T21:54:23.783ZSetup debug logging
932025-01-28T21:54:23.783Z+ mkdir /tmp/debug
942025-01-28T21:54:23.783Z+ sleep 3600
952025-01-28T21:54:23.783Z+ psrinfo -v
962025-01-28T21:54:23.787Z+ df -h
972025-01-28T21:54:23.794Z+ prstat -d d -mLc 1
982025-01-28T21:54:23.797Z+ iostat -T d -xn 1
992025-01-28T21:54:23.797Z+ mpstat -T d 1
1002025-01-28T21:54:23.797Z+ vmstat -T d -p 1
1012025-01-28T21:54:23.797Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1022025-01-28T21:54:23.800Z+ banner test_up_2r_encrypted
1032025-01-28T21:54:23.800Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1042025-01-28T21:54:23.802Z #####
1052025-01-28T21:54:23.802Z ##### ###### #### ##### # # ##### # # #####
1062025-01-28T21:54:23.802Z # # # # # # # # # # #
1072025-01-28T21:54:23.802Z # ##### #### # # # # # ##### # #
1082025-01-28T21:54:23.802Z # # # # # # ##### # #####
1092025-01-28T21:54:23.802Z # # # # # # # # # # #
1102025-01-28T21:54:23.802Z # ###### #### # ####### #### # ####### ####### # #
1112025-01-28T21:54:23.802Z
1122025-01-28T21:54:23.803Z+ ptime -m bash /input/build/work/scripts/test_up.sh -r 2 -N encrypted
1132025-01-28T21:54:23.811Z/input/build/work
1142025-01-28T21:54:23.823ZUsing 2 region sets
1152025-01-28T21:54:23.823ZTurn off color for downstairs dump
1162025-01-28T21:54:23.839ZUpstairs using key: mkpJ9bpvyun2gtD4Zt4KL8jYu3brJaf1kIDcwKbLr/s=
1172025-01-28T21:54:23.842Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1182025-01-28T21:54:23.842ZCreating 6 downstairs regions
1192025-01-28T21:54:24.200ZStarting 6 downstairs
1202025-01-28T21:54:24.202Zdsc started at PID: 1187
1212025-01-28T21:54:29.543ZDisable automatic restart on all downstairs
1222025-01-28T21:54:29.597Z
1232025-01-28T21:54:29.597ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1242025-01-28T21:54:29.601ZRunning test: span
1252025-01-28T21:54:33.773ZCompleted test: span
1262025-01-28T21:54:38.782ZRunning test: big
1272025-01-28T21:54:43.355ZCompleted test: big
1282025-01-28T21:54:48.364ZRunning test: dep
1292025-01-28T21:55:21.762ZCompleted test: dep
1302025-01-28T21:55:26.771ZRunning test: balloon
1312025-01-28T21:59:37.946ZCompleted test: balloon
1322025-01-28T21:59:42.955ZRunning test: deactivate
1332025-01-28T22:00:37.661ZCompleted test: deactivate
1342025-01-28T22:00:42.670ZRunning hammer
1352025-01-28T22:00:55.329ZRun repair tests
1362025-01-28T22:00:55.333Z/var/tmp/bins/crutest fill -g 31 -q --key mkpJ9bpvyun2gtD4Zt4KL8jYu3brJaf1kIDcwKbLr/s= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1372025-01-28T22:00:55.390Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.560163336Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"task":"crutest"}
1382025-01-28T22:00:55.393Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.560785939Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"task":"crutest"}
1392025-01-28T22:00:55.393Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.562440406Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"task":"crutest"}
1402025-01-28T22:00:55.393Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.562618293Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230}
1412025-01-28T22:00:55.393Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7817632486f84ab6865c5ce71e16196ec0c710fc\",\n git_commit_timestamp: \"2025-01-28T21:38:04.000000000Z\",\n git_branch: \"mkeeter/early-io-rejection\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.562671431Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230}
1422025-01-28T22:00:55.393Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.562698929Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230}
1432025-01-28T22:00:55.393Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.562717402Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230}
1442025-01-28T22:00:55.396Z{"msg":"Crucible 9c534274-8528-4c0a-8609-6b0a41b42d0b has session id: d222b7d5-6993-4b80-841f-47d8f911fa06","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.563359357Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1452025-01-28T22:00:55.396Z{"msg":"Upstairs opts: Upstairs UUID: 9c534274-8528-4c0a-8609-6b0a41b42d0b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.563410216Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1462025-01-28T22:00:55.396Z{"msg":"Crucible stats registered with UUID: 9c534274-8528-4c0a-8609-6b0a41b42d0b","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.563444881Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1472025-01-28T22:00:55.397Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:00:53.563466652Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1482025-01-28T22:00:55.397Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.565526702Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"task":"crutest"}
1492025-01-28T22:00:55.397Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.565596283Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230}
1502025-01-28T22:00:55.397Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7817632486f84ab6865c5ce71e16196ec0c710fc\",\n git_commit_timestamp: \"2025-01-28T21:38:04.000000000Z\",\n git_branch: \"mkeeter/early-io-rejection\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.565639275Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230}
1512025-01-28T22:00:55.397Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.565659657Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230}
1522025-01-28T22:00:55.397Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.565674331Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230}
1532025-01-28T22:00:55.397Z{"msg":"Crucible 9c534274-8528-4c0a-8609-6b0a41b42d0b has session id: 04669002-87fb-44b2-bbb6-8f1eeddc05b4","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.566037031Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
1542025-01-28T22:00:55.401Z{"msg":"Upstairs opts: Upstairs UUID: 9c534274-8528-4c0a-8609-6b0a41b42d0b, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.566086391Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
1552025-01-28T22:00:55.402Z{"msg":"Crucible stats registered with UUID: 9c534274-8528-4c0a-8609-6b0a41b42d0b","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.566118887Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
1562025-01-28T22:00:55.402Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:00:53.566139839Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
1572025-01-28T22:00:55.402Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.566736022Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"task":"crutest"}
1582025-01-28T22:00:55.402Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.566764241Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230}
1592025-01-28T22:00:55.402Z{"msg":"9c534274-8528-4c0a-8609-6b0a41b42d0b active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.566857862Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1602025-01-28T22:00:55.402Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.566941877Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"client":"0","":"downstairs","session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1612025-01-28T22:00:55.402Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.566999724Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"client":"0","":"downstairs","session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1622025-01-28T22:00:55.402Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.567038797Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"client":"1","":"downstairs","session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1632025-01-28T22:00:55.402Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.567063177Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"client":"1","":"downstairs","session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1642025-01-28T22:00:55.402Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.567085298Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"client":"2","":"downstairs","session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1652025-01-28T22:00:55.402Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.56734728Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"client":"2","":"downstairs","session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1662025-01-28T22:00:55.402Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.567428736Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1672025-01-28T22:00:55.402Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.567512572Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1682025-01-28T22:00:55.402Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.567536962Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1692025-01-28T22:00:55.402Z{"msg":"ds_connection connected from Ok(127.0.0.1:48178)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.567587121Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1702025-01-28T22:00:55.402Z{"msg":"ds_connection connected from Ok(127.0.0.1:48506)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.567634931Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1712025-01-28T22:00:55.402Z{"msg":"ds_connection connected from Ok(127.0.0.1:47660)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.56766192Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1722025-01-28T22:00:55.402Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.56785675Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"client":"1","":"downstairs","session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1732025-01-28T22:00:55.402Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.567890346Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"client":"2","":"downstairs","session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1742025-01-28T22:00:55.402Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.567905889Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"client":"0","":"downstairs","session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1752025-01-28T22:00:55.403Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.568513748Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"client":"0","":"downstairs","session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1762025-01-28T22:00:55.403Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.568549764Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"client":"1","":"downstairs","session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1772025-01-28T22:00:55.403Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.568632829Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"client":"2","":"downstairs","session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1782025-01-28T22:00:55.403Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.568948239Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1792025-01-28T22:00:55.403Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.568985823Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1802025-01-28T22:00:55.403Z{"msg":"[0]R flush_numbers: [3993, 3993, 3993, 3993, 3993]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.569009124Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"downstairs","":"downstairs","session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1812025-01-28T22:00:55.403Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.56904219Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"downstairs","":"downstairs","session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1822025-01-28T22:00:55.403Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.56906694Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"downstairs","":"downstairs","session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1832025-01-28T22:00:55.403Z{"msg":"[1]R flush_numbers: [3993, 3993, 3993, 3993, 3993]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.569097747Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"downstairs","":"downstairs","session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1842025-01-28T22:00:55.403Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.569131024Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"downstairs","":"downstairs","session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1852025-01-28T22:00:55.403Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.569156463Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"downstairs","":"downstairs","session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1862025-01-28T22:00:55.403Z{"msg":"[2]R flush_numbers: [3993, 3993, 3993, 3993, 3993]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.569200395Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"downstairs","":"downstairs","session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1872025-01-28T22:00:55.403Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.569226184Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"downstairs","":"downstairs","session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1882025-01-28T22:00:55.403Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.569265308Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"downstairs","":"downstairs","session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1892025-01-28T22:00:55.403Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.569337278Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"downstairs","":"downstairs","session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1902025-01-28T22:00:55.403Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.569438846Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"downstairs","":"downstairs","session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1912025-01-28T22:00:55.403Z{"msg":"Next flush: 3994","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.569491065Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"downstairs","":"downstairs","session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1922025-01-28T22:00:55.403Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.569522662Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"downstairs","":"downstairs","session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1932025-01-28T22:00:55.403Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.569638514Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1942025-01-28T22:00:55.403Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.569689053Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"client":"0","":"downstairs","session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1952025-01-28T22:00:55.403Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.569717861Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"client":"1","":"downstairs","session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1962025-01-28T22:00:55.404Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.569752277Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"client":"2","":"downstairs","session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1972025-01-28T22:00:55.404Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.56976796Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1982025-01-28T22:00:55.404Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.569788522Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
1992025-01-28T22:00:55.404Z{"msg":"9c534274-8528-4c0a-8609-6b0a41b42d0b is now active with session: d222b7d5-6993-4b80-841f-47d8f911fa06","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.569812562Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
2002025-01-28T22:00:55.404Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.569837722Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"session_id":"d222b7d5-6993-4b80-841f-47d8f911fa06"}
2012025-01-28T22:00:55.404Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.569861132Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230}
2022025-01-28T22:00:55.404Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.569893888Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230}
2032025-01-28T22:00:55.404Z{"msg":"9c534274-8528-4c0a-8609-6b0a41b42d0b active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.569911021Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2042025-01-28T22:00:55.404Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.569936011Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"client":"0","":"downstairs","session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2052025-01-28T22:00:55.404Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.569952364Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"client":"0","":"downstairs","session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2062025-01-28T22:00:55.404Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.569976924Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"client":"1","":"downstairs","session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2072025-01-28T22:00:55.405Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.569991708Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"client":"1","":"downstairs","session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2082025-01-28T22:00:55.405Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.57001138Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"client":"2","":"downstairs","session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2092025-01-28T22:00:55.405Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.570025334Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"client":"2","":"downstairs","session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2102025-01-28T22:00:55.405Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.570038309Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2112025-01-28T22:00:55.405Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.57005878Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2122025-01-28T22:00:55.405Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.570082311Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2132025-01-28T22:00:55.405Z{"msg":"ds_connection connected from Ok(127.0.0.1:49017)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.570120335Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2142025-01-28T22:00:55.405Z{"msg":"ds_connection connected from Ok(127.0.0.1:44292)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.570184668Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2152025-01-28T22:00:55.409Z{"msg":"ds_connection connected from Ok(127.0.0.1:50698)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.570214446Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2162025-01-28T22:00:55.409Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.57025413Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"client":"2","":"downstairs","session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2172025-01-28T22:00:55.409Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.57027624Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"client":"0","":"downstairs","session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2182025-01-28T22:00:55.409Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.570300541Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"client":"1","":"downstairs","session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2192025-01-28T22:00:55.409Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.570458745Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"client":"2","":"downstairs","session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2202025-01-28T22:00:55.409Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.570549608Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"client":"0","":"downstairs","session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2212025-01-28T22:00:55.409Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.570595569Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"client":"1","":"downstairs","session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2222025-01-28T22:00:55.409Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.570805872Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2232025-01-28T22:00:55.409Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.570941566Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2242025-01-28T22:00:55.409Z{"msg":"[0]R flush_numbers: [5314, 5313, 5313, 5314, 5314]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.571030969Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"downstairs","":"downstairs","session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2252025-01-28T22:00:55.409Z{"msg":"[0]R generation: [14, 13, 13, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.571065574Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"downstairs","":"downstairs","session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2262025-01-28T22:00:55.409Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.571081318Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"downstairs","":"downstairs","session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2272025-01-28T22:00:55.410Z{"msg":"[1]R flush_numbers: [5314, 5313, 5313, 5314, 5314]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.57110188Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"downstairs","":"downstairs","session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2282025-01-28T22:00:55.410Z{"msg":"[1]R generation: [14, 13, 13, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.571115854Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"downstairs","":"downstairs","session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2292025-01-28T22:00:55.411Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.571135786Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"downstairs","":"downstairs","session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2302025-01-28T22:00:55.411Z{"msg":"[2]R flush_numbers: [5314, 5313, 5313, 5314, 5314]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.5711495Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230Wait for a query_work_queue command to finish before sending IO
2312025-01-28T22:00:55.411Z,"":"downstairs","":"downstairs","session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2322025-01-28T22:00:55.411Z{"msg":"[2]R generation: [14, 13, 13, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.571175529Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"downstairs","":"downstairs","session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2332025-01-28T22:00:55.411Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.571202058Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"downstairs","":"downstairs","session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2342025-01-28T22:00:55.411Z{"msg":"Max found gen is 15","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.571215963Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"downstairs","":"downstairs","session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2352025-01-28T22:00:55.411Z{"msg":"Generation requested: 31 >= found:15","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.571241872Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"downstairs","":"downstairs","session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2362025-01-28T22:00:55.411Z{"msg":"Next flush: 5315","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.571260764Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"downstairs","":"downstairs","session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2372025-01-28T22:00:55.411Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.571274988Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"":"downstairs","":"downstairs","session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2382025-01-28T22:00:55.411Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.571296379Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2392025-01-28T22:00:55.411Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.571327237Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"client":"0","":"downstairs","session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2402025-01-28T22:00:55.411Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.571349357Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"client":"1","":"downstairs","session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2412025-01-28T22:00:55.411Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.571377985Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"client":"2","":"downstairs","session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2422025-01-28T22:00:55.411Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.57139257Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2432025-01-28T22:00:55.411Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.571411722Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2442025-01-28T22:00:55.411Z{"msg":"9c534274-8528-4c0a-8609-6b0a41b42d0b is now active with session: 04669002-87fb-44b2-bbb6-8f1eeddc05b4","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.571435682Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2452025-01-28T22:00:55.411Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.571453214Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230,"session_id":"04669002-87fb-44b2-bbb6-8f1eeddc05b4"}
2462025-01-28T22:00:55.411Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:53.571472187Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1230}
2472025-01-28T22:00:55.411ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
2482025-01-28T22:00:55.411ZFill test
2492025-01-28T22:00:55.444ZRead and Verify all blocks (0..100 range:false)
2502025-01-28T22:00:55.530ZWrote out file "/var/tmp/test_up-build/verify_file"
2512025-01-28T22:00:55.530ZCLIENT: Tests done. All submitted work has been ACK'd
2522025-01-28T22:00:55.533Z----------------------------------------------------------------
2532025-01-28T22:00:55.533Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2542025-01-28T22:00:55.533ZStates: Active Active Active
2552025-01-28T22:00:55.533ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2562025-01-28T22:00:55.533Z 1002 Acked Read 50 Done Done Done false
2572025-01-28T22:00:55.533Z STATES DS:0 DS:1 DS:2 TOTAL
2582025-01-28T22:00:55.533Z Sent 0 0 0 0
2592025-01-28T22:00:55.533Z Done 1 1 1 3
2602025-01-28T22:00:55.533Z Skipped 0 0 0 0
2612025-01-28T22:00:55.533Z Error 0 0 0 0
2622025-01-28T22:00:55.533ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2632025-01-28T22:00:55.533ZDownstairs last five completed: 1001 1000
2642025-01-28T22:00:55.533ZUpstairs last five completed: 1002 1001 1000
2652025-01-28T22:00:55.533Z----------------------------------------------------------------
2662025-01-28T22:00:55.533Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2672025-01-28T22:00:55.533ZStates: Active Active Active
2682025-01-28T22:00:55.533ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2692025-01-28T22:00:55.533Z 1002 Acked Read 50 Done Done Done false
2702025-01-28T22:00:55.534Z STATES DS:0 DS:1 DS:2 TOTAL
2712025-01-28T22:00:55.534Z Sent 0 0 0 0
2722025-01-28T22:00:55.534Z Done 1 1 1 3
2732025-01-28T22:00:55.534Z Skipped 0 0 0 0
2742025-01-28T22:00:55.534Z Error 0 0 0 0
2752025-01-28T22:00:55.534ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2762025-01-28T22:00:55.534ZDownstairs last five completed: 1001 1000
2772025-01-28T22:00:55.534ZUpstairs last five completed: 1002 1001 1000
2782025-01-28T22:00:55.534ZCLIENT: Up:0 ds:2 act:6
2792025-01-28T22:00:59.534Z----------------------------------------------------------------
2802025-01-28T22:00:59.534Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2812025-01-28T22:00:59.534ZDownstairs last five completed: 1003 1002 1001 1000
2822025-01-28T22:00:59.534ZUpstairs last five completed: 1003 1002 1001 1000
2832025-01-28T22:00:59.534Z----------------------------------------------------------------
2842025-01-28T22:00:59.534Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2852025-01-28T22:00:59.534ZDownstairs last five completed: 1003 1002 1001 1000
2862025-01-28T22:00:59.534ZUpstairs last five completed: 1003 1002 1001 1000
2872025-01-28T22:00:59.537ZCLIENT: Up:0 ds:0 act:6
2882025-01-28T22:00:59.537ZCLIENT: All crucible jobs finished, exiting program
2892025-01-28T22:00:59.545ZRepair setup passed
2902025-01-28T22:00:59.550ZCopy the region for /var/tmp/test_up-build/8830
2912025-01-28T22:00:59.552Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2922025-01-28T22:00:59.555Z/var/tmp/bins/crutest fill -g 32 -q --key mkpJ9bpvyun2gtD4Zt4KL8jYu3brJaf1kIDcwKbLr/s= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2932025-01-28T22:00:59.611Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.779265453Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"task":"crutest"}
2942025-01-28T22:00:59.614Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.779859378Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"task":"crutest"}
2952025-01-28T22:00:59.614Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.781498351Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"task":"crutest"}
2962025-01-28T22:00:59.614Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.781690632Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236}
2972025-01-28T22:00:59.614Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7817632486f84ab6865c5ce71e16196ec0c710fc\",\n git_commit_timestamp: \"2025-01-28T21:38:04.000000000Z\",\n git_branch: \"mkeeter/early-io-rejection\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.781739991Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236}
2982025-01-28T22:00:59.614Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.78176659Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236}
2992025-01-28T22:00:59.614Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.781785093Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236}
3002025-01-28T22:00:59.617Z{"msg":"Crucible 500b52a2-93a7-475c-8d1c-6da23dd8aeb8 has session id: 68832402-30fb-4240-b061-be9582c3dea3","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.782369341Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3012025-01-28T22:00:59.617Z{"msg":"Upstairs opts: Upstairs UUID: 500b52a2-93a7-475c-8d1c-6da23dd8aeb8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.782411424Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3022025-01-28T22:00:59.617Z{"msg":"Crucible stats registered with UUID: 500b52a2-93a7-475c-8d1c-6da23dd8aeb8","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.782455985Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3032025-01-28T22:00:59.617Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:00:57.782478256Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3042025-01-28T22:00:59.617Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.78440623Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"task":"crutest"}
3052025-01-28T22:00:59.617Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.784494194Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236}
3062025-01-28T22:00:59.617Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7817632486f84ab6865c5ce71e16196ec0c710fc\",\n git_commit_timestamp: \"2025-01-28T21:38:04.000000000Z\",\n git_branch: \"mkeeter/early-io-rejection\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.784537836Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236}
3072025-01-28T22:00:59.617Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.784561846Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236}
3082025-01-28T22:00:59.617Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.784579659Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236}
3092025-01-28T22:00:59.617Z{"msg":"Crucible 500b52a2-93a7-475c-8d1c-6da23dd8aeb8 has session id: 75220029-2378-4d32-8b71-3e1ad305b93e","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.784922307Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3102025-01-28T22:00:59.620Z{"msg":"Upstairs opts: Upstairs UUID: 500b52a2-93a7-475c-8d1c-6da23dd8aeb8, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.78498856Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3112025-01-28T22:00:59.620Z{"msg":"Crucible stats registered with UUID: 500b52a2-93a7-475c-8d1c-6da23dd8aeb8","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.785049305Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3122025-01-28T22:00:59.620Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:00:57.785080841Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3132025-01-28T22:00:59.620Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.785694148Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"task":"crutest"}
3142025-01-28T22:00:59.620Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.785735931Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236}
3152025-01-28T22:00:59.620Z{"msg":"500b52a2-93a7-475c-8d1c-6da23dd8aeb8 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.785799765Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3162025-01-28T22:00:59.620Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.785836949Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"client":"0","":"downstairs","session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3172025-01-28T22:00:59.620Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.785882561Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"client":"0","":"downstairs","session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3182025-01-28T22:00:59.620Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.785910019Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"client":"1","":"downstairs","session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3192025-01-28T22:00:59.620Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.785937748Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"client":"1","":"downstairs","session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3202025-01-28T22:00:59.620Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.785975072Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"client":"2","":"downstairs","session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3212025-01-28T22:00:59.620Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.786016965Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"client":"2","":"downstairs","session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3222025-01-28T22:00:59.620Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.786046893Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"io task","client":"2","":"downstairs","session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3232025-01-28T22:00:59.620Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.786088815Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"io task","client":"0","":"downstairs","session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3242025-01-28T22:00:59.620Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.786187135Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"io task","client":"1","":"downstairs","session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3252025-01-28T22:00:59.620Z{"msg":"ds_connection connected from Ok(127.0.0.1:47520)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.786310634Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"io task","client":"2","":"downstairs","session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3262025-01-28T22:00:59.620Z{"msg":"ds_connection connected from Ok(127.0.0.1:63901)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.78636878Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"io task","client":"0","":"downstairs","session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3272025-01-28T22:00:59.620Z{"msg":"ds_connection connected from Ok(127.0.0.1:35133)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.786396838Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"io task","client":"1","":"downstairs","session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3282025-01-28T22:00:59.620Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.786778301Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"client":"1","":"downstairs","session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3292025-01-28T22:00:59.620Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.786825691Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"client":"0","":"downstairs","session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3302025-01-28T22:00:59.621Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.786897891Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"client":"2","":"downstairs","session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3312025-01-28T22:00:59.621Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.787455531Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"client":"1","":"downstairs","session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3322025-01-28T22:00:59.621Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.787498003Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"client":"2","":"downstairs","session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3332025-01-28T22:00:59.621Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.787594444Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"client":"0","":"downstairs","session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3342025-01-28T22:00:59.621Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.787889772Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3352025-01-28T22:00:59.621Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.787924318Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3362025-01-28T22:00:59.621Z{"msg":"[0]R flush_numbers: [3994, 3994, 3994, 3994, 3994]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.787963381Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"downstairs","":"downstairs","session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3372025-01-28T22:00:59.621Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.787984782Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"downstairs","":"downstairs","session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3382025-01-28T22:00:59.621Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.787998687Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"downstairs","":"downstairs","session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3392025-01-28T22:00:59.621Z{"msg":"[1]R flush_numbers: [3994, 3994, 3994, 3994, 3994]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.788018608Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"downstairs","":"downstairs","session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3402025-01-28T22:00:59.621Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.788032023Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"downstairs","":"downstairs","session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3412025-01-28T22:00:59.621Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.788051375Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"downstairs","":"downstairs","session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3422025-01-28T22:00:59.621Z{"msg":"[2]R flush_numbers: [3994, 3994, 3994, 3994, 3994]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.788064989Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"downstairs","":"downstairs","session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3432025-01-28T22:00:59.621Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.788084631Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"downstairs","":"downstairs","session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3442025-01-28T22:00:59.624Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.788097986Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"downstairs","":"downstairs","session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3452025-01-28T22:00:59.624Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.788110781Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"downstairs","":"downstairs","session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3462025-01-28T22:00:59.624Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.788131262Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"downstairs","":"downstairs","session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3472025-01-28T22:00:59.624Z{"msg":"Next flush: 3995","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.788144946Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"downstairs","":"downstairs","session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3482025-01-28T22:00:59.624Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.788158241Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"downstairs","":"downstairs","session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3492025-01-28T22:00:59.624Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.788171116Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3502025-01-28T22:00:59.624Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.788190497Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"client":"0","":"downstairs","session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3512025-01-28T22:00:59.625Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.788203852Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"client":"1","":"downstairs","session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3522025-01-28T22:00:59.625Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.788216427Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"client":"2","":"downstairs","session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3532025-01-28T22:00:59.625Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.788229332Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3542025-01-28T22:00:59.625Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.788241167Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3552025-01-28T22:00:59.625Z{"msg":"500b52a2-93a7-475c-8d1c-6da23dd8aeb8 is now active with session: 68832402-30fb-4240-b061-be9582c3dea3","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.788261188Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3562025-01-28T22:00:59.625Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.788273643Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"session_id":"68832402-30fb-4240-b061-be9582c3dea3"}
3572025-01-28T22:00:59.625Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.788285618Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236}
3582025-01-28T22:00:59.625Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.788310208Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236}
3592025-01-28T22:00:59.625Z{"msg":"500b52a2-93a7-475c-8d1c-6da23dd8aeb8 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.788339856Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3602025-01-28T22:00:59.625Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.788372193Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"client":"0","":"downstairs","session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3612025-01-28T22:00:59.625Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.7884021Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"client":"0","":"downstairs","session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3622025-01-28T22:00:59.626Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.78842589Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"client":"1","":"downstairs","session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3632025-01-28T22:00:59.626Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.788463365Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"client":"1","":"downstairs","session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3642025-01-28T22:00:59.626Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.788612173Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"client":"2","":"downstairs","session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3652025-01-28T22:00:59.626Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.788718949Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"client":"2","":"downstairs","session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3662025-01-28T22:00:59.626Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.788760082Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"io task","client":"2","":"downstairs","session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3672025-01-28T22:00:59.626Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.788787231Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"io task","client":"0","":"downstairs","session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3682025-01-28T22:00:59.626Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.788821057Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"io task","client":"1","":"downstairs","session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3692025-01-28T22:00:59.626Z{"msg":"ds_connection connected from Ok(127.0.0.1:52710)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.788847536Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"io task","client":"2","":"downstairs","session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3702025-01-28T22:00:59.626Z{"msg":"ds_connection connected from Ok(127.0.0.1:46299)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.788873885Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"io task","client":"0","":"downstairs","session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3712025-01-28T22:00:59.626Z{"msg":"ds_connection connected from Ok(127.0.0.1:40111)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.788912809Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"io task","client":"1","":"downstairs","session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3722025-01-28T22:00:59.626Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.788938489Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"client":"0","":"downstairs","session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3732025-01-28T22:00:59.626Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.78908022Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"client":"1","":"downstairs","session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3742025-01-28T22:00:59.626Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.789109148Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"client":"2","":"downstairs","session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3752025-01-28T22:00:59.626Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.78939453Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"client":"0","":"downstairs","session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3762025-01-28T22:00:59.626Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.789768026Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"client":"1","":"downstairs","session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3772025-01-28T22:00:59.626Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.789813767Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"client":"2","":"downstairs","session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3782025-01-28T22:00:59.626Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.789842555Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3792025-01-28T22:00:59.626Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.790113074Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3802025-01-28T22:00:59.626Z{"msg":"[0]R flush_numbers: [5315, 5315, 5315, 5315, 5315]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.79014581Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"downstairs","":"downstairs","session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3812025-01-28T22:00:59.626Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.790163473Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"downstairs","":"downstairs","session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3822025-01-28T22:00:59.627Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.790197759Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"downstairs","":"downstairs","session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3832025-01-28T22:00:59.627Z{"msg":"[1]R flush_numbers: [5315, 5315, 5315, 5315, 5315]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
3842025-01-28T22:00:59.627Z,"time":"2025-01-28T22:00:57.790232604Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"downstairs","":"downstairs","session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3852025-01-28T22:00:59.627Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.790268829Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"downstairs","":"downstairs","session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3862025-01-28T22:00:59.627Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.790303515Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"downstairs","":"downstairs","session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3872025-01-28T22:00:59.627Z{"msg":"[2]R flush_numbers: [5315, 5315, 5315, 5315, 5315]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.79033929Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"downstairs","":"downstairs","session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3882025-01-28T22:00:59.627Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.790379653Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"downstairs","":"downstairs","session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3892025-01-28T22:00:59.627Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.790420547Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"downstairs","":"downstairs","session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3902025-01-28T22:00:59.627Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.790469706Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"downstairs","":"downstairs","session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3912025-01-28T22:00:59.627Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.79050836Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"downstairs","":"downstairs","session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3922025-01-28T22:00:59.627Z{"msg":"Next flush: 5316","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.790545775Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"downstairs","":"downstairs","session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3932025-01-28T22:00:59.629Z{"msg":"All extents match","v":0,"name":"crucible","level":30Disk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
3942025-01-28T22:00:59.629Z,"time":"2025-01-28T22:00:57.79058256Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"":"downstairs","":"downstairs","session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3952025-01-28T22:00:59.629Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.790617495Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3962025-01-28T22:00:59.629Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.790632419Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"client":"0","":"downstairs","session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3972025-01-28T22:00:59.629Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.7906539Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"client":"1","":"downstairs","session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3982025-01-28T22:00:59.629Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.790667854Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"client":"2","":"downstairs","session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
3992025-01-28T22:00:59.629Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.790687646Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
4002025-01-28T22:00:59.629Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.790699941Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
4012025-01-28T22:00:59.629Z{"msg":"500b52a2-93a7-475c-8d1c-6da23dd8aeb8 is now active with session: 75220029-2378-4d32-8b71-3e1ad305b93e","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.790718824Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
4022025-01-28T22:00:59.629Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.790730859Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236,"session_id":"75220029-2378-4d32-8b71-3e1ad305b93e"}
4032025-01-28T22:00:59.629ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4042025-01-28T22:00:59.629Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:00:57.790754479Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1236}
4052025-01-28T22:00:59.629ZFill test
4062025-01-28T22:00:59.664ZRead and Verify all blocks (0..100 range:false)
4072025-01-28T22:00:59.748ZWrote out file "/var/tmp/test_up-build/verify_file"
4082025-01-28T22:00:59.748ZCLIENT: Tests done. All submitted work has been ACK'd
4092025-01-28T22:00:59.751Z----------------------------------------------------------------
4102025-01-28T22:00:59.751Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4112025-01-28T22:00:59.751ZStates: Active Active Active
4122025-01-28T22:00:59.751ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4132025-01-28T22:00:59.751Z 1002 Acked Read 50 Done Done Done false
4142025-01-28T22:00:59.751Z STATES DS:0 DS:1 DS:2 TOTAL
4152025-01-28T22:00:59.751Z Sent 0 0 0 0
4162025-01-28T22:00:59.751Z Done 1 1 1 3
4172025-01-28T22:00:59.751Z Skipped 0 0 0 0
4182025-01-28T22:00:59.751Z Error 0 0 0 0
4192025-01-28T22:00:59.751ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4202025-01-28T22:00:59.751ZDownstairs last five completed: 1001 1000
4212025-01-28T22:00:59.751ZUpstairs last five completed: 1002 1001 1000
4222025-01-28T22:00:59.751Z----------------------------------------------------------------
4232025-01-28T22:00:59.751Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4242025-01-28T22:00:59.752ZStates: Active Active Active
4252025-01-28T22:00:59.752ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4262025-01-28T22:00:59.752Z 1002 Acked Read 50 Done Done Done false
4272025-01-28T22:00:59.752Z STATES DS:0 DS:1 DS:2 TOTAL
4282025-01-28T22:00:59.752Z Sent 0 0 0 0
4292025-01-28T22:00:59.753Z Done 1 1 1 3
4302025-01-28T22:00:59.753Z Skipped 0 0 0 0
4312025-01-28T22:00:59.753Z Error 0 0 0 0
4322025-01-28T22:00:59.753ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4332025-01-28T22:00:59.753ZDownstairs last five completed: 1001 1000
4342025-01-28T22:00:59.753ZUpstairs last five completed: 1002 1001 1000
4352025-01-28T22:00:59.753ZCLIENT: Up:0 ds:2 act:6
4362025-01-28T22:01:03.751Z----------------------------------------------------------------
4372025-01-28T22:01:03.751Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4382025-01-28T22:01:03.751ZDownstairs last five completed: 1003 1002 1001 1000
4392025-01-28T22:01:03.751ZUpstairs last five completed: 1003 1002 1001 1000
4402025-01-28T22:01:03.753Z----------------------------------------------------------------
4412025-01-28T22:01:03.753Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4422025-01-28T22:01:03.753ZDownstairs last five completed: 1003 1002 1001 1000
4432025-01-28T22:01:03.753ZUpstairs last five completed: 1003 1002 1001 1000
4442025-01-28T22:01:03.753ZCLIENT: Up:0 ds:0 act:6
4452025-01-28T22:01:03.753ZCLIENT: All crucible jobs finished, exiting program
4462025-01-28T22:01:03.758ZRepair part 1 passed
4472025-01-28T22:01:03.761Z
4482025-01-28T22:01:03.761ZKill the current downstairs
4492025-01-28T22:01:03.926ZDownstairs 2 stopped
4502025-01-28T22:01:03.926Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
4512025-01-28T22:01:03.929ZNow put back the original so we have a mismatch
4522025-01-28T22:01:03.929Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
4532025-01-28T22:01:03.932ZRestart downstairs with old directory
4542025-01-28T22:01:03.985Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4552025-01-28T22:01:04.002ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4562025-01-28T22:01:04.005Z 0 000-009 32 32 31 3995 3995 3994 F F F <---
4572025-01-28T22:01:04.005Z 1 010-019 32 32 31 3995 3995 3994 F F F <---
4582025-01-28T22:01:04.005Z 2 020-029 32 32 31 3995 3995 3994 F F F <---
4592025-01-28T22:01:04.005Z 3 030-039 32 32 31 3995 3995 3994 F F F <---
4602025-01-28T22:01:04.005Z 4 040-049 32 32 31 3995 3995 3994 F F F <---
4612025-01-28T22:01:04.005ZMax gen: 32, Max flush: 3995
4622025-01-28T22:01:04.005ZError: Difference in extent metadata found!
4632025-01-28T22:01:04.008Zdump test found error as expected
4642025-01-28T22:01:04.008Z
4652025-01-28T22:01:04.008Z
4662025-01-28T22:01:04.008Z/var/tmp/bins/crutest verify --range -g 33 -q --key mkpJ9bpvyun2gtD4Zt4KL8jYu3brJaf1kIDcwKbLr/s= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
4672025-01-28T22:01:04.065Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.229872547Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"task":"crutest"}
4682025-01-28T22:01:04.069Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.230476867Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"task":"crutest"}
4692025-01-28T22:01:04.069Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.232200715Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"task":"crutest"}
4702025-01-28T22:01:04.069Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.232385059Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244}
4712025-01-28T22:01:04.069Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7817632486f84ab6865c5ce71e16196ec0c710fc\",\n git_commit_timestamp: \"2025-01-28T21:38:04.000000000Z\",\n git_branch: \"mkeeter/early-io-rejection\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.232443125Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244}
4722025-01-28T22:01:04.069Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.23248006Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244}
4732025-01-28T22:01:04.070Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.232500142Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244}
4742025-01-28T22:01:04.070Z{"msg":"Crucible f6c6a568-3f57-4c9b-8643-cbdcfa142aab has session id: 207bf205-d7c1-4d04-9659-c87ff889dd9c","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.233048275Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
4752025-01-28T22:01:04.070Z{"msg":"Upstairs opts: Upstairs UUID: f6c6a568-3f57-4c9b-8643-cbdcfa142aab, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.23310792Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
4762025-01-28T22:01:04.070Z{"msg":"Crucible stats registered with UUID: f6c6a568-3f57-4c9b-8643-cbdcfa142aab","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.23313322Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
4772025-01-28T22:01:04.070Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:01:02.233163288Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
4782025-01-28T22:01:04.070Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.235222227Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"task":"crutest"}
4792025-01-28T22:01:04.070Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.235310491Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244}
4802025-01-28T22:01:04.070Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7817632486f84ab6865c5ce71e16196ec0c710fc\",\n git_commit_timestamp: \"2025-01-28T21:38:04.000000000Z\",\n git_branch: \"mkeeter/early-io-rejection\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.235346226Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244}
4812025-01-28T22:01:04.070Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.235360521Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244}
4822025-01-28T22:01:04.070Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.235375004Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244}
4832025-01-28T22:01:04.075Z{"msg":"Crucible f6c6a568-3f57-4c9b-8643-cbdcfa142aab has session id: 30015cf7-d9f5-4df6-8d02-c464da7ed69b","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.235804847Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
4842025-01-28T22:01:04.075Z{"msg":"Upstairs opts: Upstairs UUID: f6c6a568-3f57-4c9b-8643-cbdcfa142aab, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.235859364Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
4852025-01-28T22:01:04.075Z{"msg":"Crucible stats registered with UUID: f6c6a568-3f57-4c9b-8643-cbdcfa142aab","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.235885634Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
4862025-01-28T22:01:04.075Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:01:02.235923968Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
4872025-01-28T22:01:04.075Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.236588083Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"task":"crutest"}
4882025-01-28T22:01:04.075Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.236641072Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244}
4892025-01-28T22:01:04.075Z{"msg":"f6c6a568-3f57-4c9b-8643-cbdcfa142aab active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.236724027Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
4902025-01-28T22:01:04.075Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.236810532Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"0","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
4912025-01-28T22:01:04.075Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.236875315Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"0","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
4922025-01-28T22:01:04.075Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.236938379Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"1","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
4932025-01-28T22:01:04.075Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.236965578Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"1","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
4942025-01-28T22:01:04.075Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.236997474Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"2","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
4952025-01-28T22:01:04.075Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.237021415Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"2","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
4962025-01-28T22:01:04.075Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.237042926Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
4972025-01-28T22:01:04.075Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.237065516Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
4982025-01-28T22:01:04.075Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.237101802Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
4992025-01-28T22:01:04.075Z{"msg":"ds_connection connected from Ok(127.0.0.1:44941)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.237172912Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5002025-01-28T22:01:04.075Z{"msg":"ds_connection connected from Ok(127.0.0.1:61716)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.237261136Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5012025-01-28T22:01:04.075Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:01:02.237282317Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5022025-01-28T22:01:04.075Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:01:02.237312504Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5032025-01-28T22:01:04.075Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:01:02.237328668Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5042025-01-28T22:01:04.075Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.237340843Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5052025-01-28T22:01:04.075Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.237777432Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"0","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5062025-01-28T22:01:04.076Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.237864027Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"1","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5072025-01-28T22:01:04.077Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.238380873Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"0","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5082025-01-28T22:01:04.077Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.238487999Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"1","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5092025-01-28T22:01:04.077Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.238729629Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5102025-01-28T22:01:04.077Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:02.238805358Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5112025-01-28T22:01:14.077Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.238284684Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5122025-01-28T22:01:14.082Z{"msg":"ds_connection connected from Ok(127.0.0.1:40938)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.238454493Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5132025-01-28T22:01:14.082Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.239594013Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"2","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5142025-01-28T22:01:14.082Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.240154001Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"2","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5152025-01-28T22:01:14.082Z{"msg":"[0]R flush_numbers: [3995, 3995, 3995, 3995, 3995]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.240548718Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"downstairs","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5162025-01-28T22:01:14.082Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.240581955Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"downstairs","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5172025-01-28T22:01:14.082Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.240597409Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"downstairs","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5182025-01-28T22:01:14.082Z{"msg":"[1]R flush_numbers: [3995, 3995, 3995, 3995, 3995]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.240619849Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"downstairs","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5192025-01-28T22:01:14.082Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.24064121Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"downstairs","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5202025-01-28T22:01:14.082Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.240656614Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"downstairs","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5212025-01-28T22:01:14.082Z{"msg":"[2]R flush_numbers: [3994, 3994, 3994, 3994, 3994]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.240670368Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"downstairs","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5222025-01-28T22:01:14.082Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.24069025Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"downstairs","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5232025-01-28T22:01:14.082Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.240704025Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"downstairs","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5242025-01-28T22:01:14.082Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.240720248Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"downstairs","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5252025-01-28T22:01:14.082Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.240744908Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"downstairs","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5262025-01-28T22:01:14.082Z{"msg":"Next flush: 3996","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.24076181Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"downstairs","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5272025-01-28T22:01:14.082Z{"msg":"extent 0 needs reconciliation: ExtentMetadata { gen: 32, flush: 3995, dirty: false } ExtentMetadata { gen: 32, flush: 3995, dirty: false } ExtentMetadata { gen: 31, flush: 3994, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.240788619Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5282025-01-28T22:01:14.082Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 3995, dirty: false }, ExtentMetadata { gen: 32, flush: 3995, dirty: false }, ExtentMetadata { gen: 31, flush: 3994, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.240816458Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5292025-01-28T22:01:14.083Z{"msg":"found dest for source 0 for extent at index 0 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.240834391Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5302025-01-28T22:01:14.083Z{"msg":"extent 1 needs reconciliation: ExtentMetadata { gen: 32, flush: 3995, dirty: false } ExtentMetadata { gen: 32, flush: 3995, dirty: false } ExtentMetadata { gen: 31, flush: 3994, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.240861749Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5312025-01-28T22:01:14.083Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 3995, dirty: false }, ExtentMetadata { gen: 32, flush: 3995, dirty: false }, ExtentMetadata { gen: 31, flush: 3994, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.240881651Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5322025-01-28T22:01:14.083Z{"msg":"found dest for source 0 for extent at index 1 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.240906861Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5332025-01-28T22:01:14.083Z{"msg":"extent 2 needs reconciliation: ExtentMetadata { gen: 32, flush: 3995, dirty: false } ExtentMetadata { gen: 32, flush: 3995, dirty: false } ExtentMetadata { gen: 31, flush: 3994, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.240926303Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5342025-01-28T22:01:14.083Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 3995, dirty: false }, ExtentMetadata { gen: 32, flush: 3995, dirty: false }, ExtentMetadata { gen: 31, flush: 3994, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.240953091Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5352025-01-28T22:01:14.083Z{"msg":"found dest for source 0 for extent at index 2 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.240971274Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5362025-01-28T22:01:14.083Z{"msg":"extent 3 needs reconciliation: ExtentMetadata { gen: 32, flush: 3995, dirty: false } ExtentMetadata { gen: 32, flush: 3995, dirty: false } ExtentMetadata { gen: 31, flush: 3994, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.240998083Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5372025-01-28T22:01:14.083Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 3995, dirty: false }, ExtentMetadata { gen: 32, flush: 3995, dirty: false }, ExtentMetadata { gen: 31, flush: 3994, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.241024722Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5382025-01-28T22:01:14.083Z{"msg":"found dest for source 0 for extent at index 3 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.241060287Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5392025-01-28T22:01:14.083Z{"msg":"extent 4 needs reconciliation: ExtentMetadata { gen: 32, flush: 3995, dirty: false } ExtentMetadata { gen: 32, flush: 3995, dirty: false } ExtentMetadata { gen: 31, flush: 3994, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.241098622Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5402025-01-28T22:01:14.083Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 3995, dirty: false }, ExtentMetadata { gen: 32, flush: 3995, dirty: false }, ExtentMetadata { gen: 31, flush: 3994, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.24112741Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5412025-01-28T22:01:14.083Z{"msg":"found dest for source 0 for extent at index 4 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.241171851Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5422025-01-28T22:01:14.083Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.241226998Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"0","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5432025-01-28T22:01:14.083Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.2412717Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"1","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5442025-01-28T22:01:14.083Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.241298369Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"2","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5452025-01-28T22:01:14.083Z{"msg":"Full repair list: {ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.241340831Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"downstairs","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5462025-01-28T22:01:14.083Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 3996, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 3996, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 3996, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 3996, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 3996, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.241477345Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"downstairs","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5472025-01-28T22:01:14.083Z{"msg":"starting reconciliation 232faec9-0c5e-42db-9e25-31072bbf7d8b: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.241502435Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"downstairs","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5482025-01-28T22:01:14.083Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.241519837Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"2","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5492025-01-28T22:01:14.089Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.251039557Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"2","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5502025-01-28T22:01:14.093Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.254790678Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"2","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5512025-01-28T22:01:14.096Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.258317901Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"2","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5522025-01-28T22:01:14.100Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.262086795Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"2","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5532025-01-28T22:01:14.104Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.265196981Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"downstairs","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5542025-01-28T22:01:14.104Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.265232656Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"0","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5552025-01-28T22:01:14.104Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.265248459Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"1","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5562025-01-28T22:01:14.104Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.265261924Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"2","":"downstairs","session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5572025-01-28T22:01:14.104Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.265275648Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5582025-01-28T22:01:14.104Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.265288193Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5592025-01-28T22:01:14.104Z{"msg":"f6c6a568-3f57-4c9b-8643-cbdcfa142aab is now active with session: 207bf205-d7c1-4d04-9659-c87ff889dd9c","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.265300898Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"session_id":"207bf205-d7c1-4d04-9659-c87ff889dd9c"}
5602025-01-28T22:01:14.104Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.265315162Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244}
5612025-01-28T22:01:14.104Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.265326877Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244}
5622025-01-28T22:01:14.104Z{"msg":"f6c6a568-3f57-4c9b-8643-cbdcfa142aab active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.265338102Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5632025-01-28T22:01:14.104Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.265350037Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"0","":"downstairs","session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5642025-01-28T22:01:14.104Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.265373768Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"0","":"downstairs","session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5652025-01-28T22:01:14.104Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.265413182Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"1","":"downstairs","session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5662025-01-28T22:01:14.107Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.265438311Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"1","":"downstairs","session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5672025-01-28T22:01:14.107Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.265469048Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"2","":"downstairs","session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5682025-01-28T22:01:14.107Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.265493438Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"2","":"downstairs","session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5692025-01-28T22:01:14.107Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.265517778Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5702025-01-28T22:01:14.107Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.265541069Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5712025-01-28T22:01:14.107Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.265565399Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5722025-01-28T22:01:14.107Z{"msg":"ds_connection connected from Ok(127.0.0.1:42606)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.265591598Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5732025-01-28T22:01:14.107Z{"msg":"ds_connection connected from Ok(127.0.0.1:57312)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.265619206Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5742025-01-28T22:01:14.107Z{"msg":"ds_connection connected from Ok(127.0.0.1:38873)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.265646025Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5752025-01-28T22:01:14.107Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.265943922Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"1","":"downstairs","session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5762025-01-28T22:01:14.108Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.265980927Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"2","":"downstairs","session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5772025-01-28T22:01:14.108Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.266103426Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"0","":"downstairs","session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5782025-01-28T22:01:14.108Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.266547783Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"1","":"downstairs","session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5792025-01-28T22:01:14.108Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.266589146Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"2","":"downstairs","session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5802025-01-28T22:01:14.108Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.266617734Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"0","":"downstairs","session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5812025-01-28T22:01:14.108Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.266923208Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5822025-01-28T22:01:14.108Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.26696693Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5832025-01-28T22:01:14.108Z{"msg":"[0]R flush_numbers: [5316, 5316, 5316, 5316, 5316]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.266985762Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"downstairs","":"downstairs","session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5842025-01-28T22:01:14.108Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.267000836Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"downstairs","":"downstairs","session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5852025-01-28T22:01:14.108Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.26701544Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"downstairs","":"downstairs","session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5862025-01-28T22:01:14.108Z{"msg":"[1]R flush_numbers: [5316, 5316, 5316, 5316, 5316]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.267028724Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"downstairs","":"downstairs","session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5872025-01-28T22:01:14.108Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.267050635Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"downstairs","":"downstairs","session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5882025-01-28T22:01:14.108Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.267064999Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"downstairs","":"downstairs","session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5892025-01-28T22:01:14.108Z{"msg":"[2]R flush_numbers: [5316, 5316, 5316, 5316, 5316]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.267078304Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"downstairs","":"downstairs","session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5902025-01-28T22:01:14.108Z{"msg":"[2]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
5912025-01-28T22:01:14.108Z,"time":"2025-01-28T22:01:12.267099135Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"downstairs","":"downstairs","session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5922025-01-28T22:01:14.108Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.267116628Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"downstairs","":"downstairs","session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5932025-01-28T22:01:14.108Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.267129683Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"downstairs","":"downstairs","session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5942025-01-28T22:01:14.108Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.267151474Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"downstairs","":"downstairs","session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5952025-01-28T22:01:14.109Z{"msg":"Next flush: 5317","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.267187349Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"downstairs","":"downstairs","session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5962025-01-28T22:01:14.109Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.267222164Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"":"downstairs","":"downstairs","session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5972025-01-28T22:01:14.109Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.26725712Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5982025-01-28T22:01:14.109Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.267292505Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"0","":"downstairs","session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
5992025-01-28T22:01:14.109Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.26732848Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"1","":"downstairs","session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
6002025-01-28T22:01:14.109Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.267367095Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"client":"2","":"downstairs","session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
6012025-01-28T22:01:14.109Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.26740364Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
6022025-01-28T22:01:14.109Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.267440354Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
6032025-01-28T22:01:14.109ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
6042025-01-28T22:01:14.109Z{"msg":"f6c6a568-3f57-4c9b-8643-cbdcfa142aab is now active with session: 30015cf7-d9f5-4df6-8d02-c464da7ed69b","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.267482187Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
6052025-01-28T22:01:14.109Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.267505507Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244,"session_id":"30015cf7-d9f5-4df6-8d02-c464da7ed69b"}
6062025-01-28T22:01:14.109Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:01:12.267519042Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244}
6072025-01-28T22:01:14.109ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
6082025-01-28T22:01:14.109ZRead and Verify all blocks (0..100 range:true)
6092025-01-28T22:01:14.184ZWrote out file "/var/tmp/test_up-build/verify_file"
6102025-01-28T22:01:14.184ZVerify test completed
6112025-01-28T22:01:14.187ZWrote out file "/var/tmp/test_up-build/verify_file"
6122025-01-28T22:01:14.187ZCLIENT: Tests done. All submitted work has been ACK'd
6132025-01-28T22:01:14.187Z----------------------------------------------------------------
6142025-01-28T22:01:14.187Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6152025-01-28T22:01:14.187ZStates: Active Active Active
6162025-01-28T22:01:14.187ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6172025-01-28T22:01:14.187Z 1000 Acked Read 50 Done Done Done false
6182025-01-28T22:01:14.187Z STATES DS:0 DS:1 DS:2 TOTAL
6192025-01-28T22:01:14.187Z Sent 0 0 0 0
6202025-01-28T22:01:14.187Z Done 1 1 1 3
6212025-01-28T22:01:14.187Z Skipped 0 0 0 0
6222025-01-28T22:01:14.187Z Error 0 0 0 0
6232025-01-28T22:01:14.187ZLast Flush: None None None
6242025-01-28T22:01:14.187ZDownstairs last five completed:
6252025-01-28T22:01:14.187ZUpstairs last five completed: 1000
6262025-01-28T22:01:14.187Z----------------------------------------------------------------
6272025-01-28T22:01:14.187Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6282025-01-28T22:01:14.187ZStates: Active Active Active
6292025-01-28T22:01:14.187ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6302025-01-28T22:01:14.187Z 1000 Acked Read 50 Done Sent Sent false
6312025-01-28T22:01:14.187Z STATES DS:0 DS:1 DS:2 TOTAL
6322025-01-28T22:01:14.187Z Sent 0 1 1 2
6332025-01-28T22:01:14.187Z Done 1 0 0 1
6342025-01-28T22:01:14.187Z Skipped 0 0 0 0
6352025-01-28T22:01:14.187Z Error 0 0 0 0
6362025-01-28T22:01:14.187ZLast Flush: None None None
6372025-01-28T22:01:14.188ZDownstairs last five completed:
6382025-01-28T22:01:14.188ZUpstairs last five completed: 1000
6392025-01-28T22:01:14.188ZCLIENT: Up:0 ds:2 act:6
6402025-01-28T22:01:18.186Z----------------------------------------------------------------
6412025-01-28T22:01:18.186Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6422025-01-28T22:01:18.186ZDownstairs last five completed: 1001 1000
6432025-01-28T22:01:18.186ZUpstairs last five completed: 1001 1000
6442025-01-28T22:01:18.186Z----------------------------------------------------------------
6452025-01-28T22:01:18.186Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6462025-01-28T22:01:18.186ZDownstairs last five completed: 1001 1000
6472025-01-28T22:01:18.186ZUpstairs last five completed: 1001 1000
6482025-01-28T22:01:18.189ZCLIENT: Up:0 ds:0 act:6
6492025-01-28T22:01:18.190ZCLIENT: All crucible jobs finished, exiting program
6502025-01-28T22:01:18.190Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:01:16.346406258Z","hostname":"w-01JJQEEPB67WVEEERWJF95B088","pid":1244}
6512025-01-28T22:01:18.193ZRepair part 2 passed
6522025-01-28T22:01:18.196Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
6532025-01-28T22:01:18.211ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6542025-01-28T22:01:18.211Z 0 000-009 32 32 32 3995 3995 3995 F F F
6552025-01-28T22:01:18.211Z 1 010-019 32 32 32 3995 3995 3995 F F F
6562025-01-28T22:01:18.211Z 2 020-029 32 32 32 3995 3995 3995 F F F
6572025-01-28T22:01:18.214Z 3 030-039 32 32 32 3995 3995 3995 F F F
6582025-01-28T22:01:18.214Z 4 040-049 32 32 32 3995 3995 3995 F F F
6592025-01-28T22:01:18.214ZMax gen: 32, Max flush: 3995
6602025-01-28T22:01:18.217Zdump test passed
6612025-01-28T22:01:18.217Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
6622025-01-28T22:01:18.231Z Extent 1
6632025-01-28T22:01:18.234ZGEN 32 32 32
6642025-01-28T22:01:18.234ZFLUSH_ID 3995 3995 3995
6652025-01-28T22:01:18.234ZDIRTY
6662025-01-28T22:01:18.234Z
6672025-01-28T22:01:18.234ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
6682025-01-28T22:01:18.234Z010 A A A A A A
6692025-01-28T22:01:18.237Z011 A A A A A A
6702025-01-28T22:01:18.237Z012 A A A A A A
6712025-01-28T22:01:18.239Z013 A A A A A A
6722025-01-28T22:01:18.239Z014 A A A A A A
6732025-01-28T22:01:18.242Z015 A A A A A A
6742025-01-28T22:01:18.242Z016 A A A A A A
6752025-01-28T22:01:18.245Z017 A A A A A A
6762025-01-28T22:01:18.245Z018 A A A A A A
6772025-01-28T22:01:18.247Z019 A A A A A A
6782025-01-28T22:01:18.251Zdump extent test passed
6792025-01-28T22:01:18.254Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
6802025-01-28T22:01:18.270ZExtent 2 Block in extent 0 Actual block 20
6812025-01-28T22:01:18.270Z
6822025-01-28T22:01:18.272Z DATA SHA256 VER
6832025-01-28T22:01:18.272Z------ ---------------------------------------------------------------- ---
6842025-01-28T22:01:18.272Z 0 00d6fe019503bba030f5f1ed1bc5cb1461de6f8aa38bb1f268c3c2b015666a17 A
6852025-01-28T22:01:18.272Z 1 00d6fe019503bba030f5f1ed1bc5cb1461de6f8aa38bb1f268c3c2b015666a17 A
6862025-01-28T22:01:18.275Z 2 00d6fe019503bba030f5f1ed1bc5cb1461de6f8aa38bb1f268c3c2b015666a17 A
6872025-01-28T22:01:18.275Z
6882025-01-28T22:01:18.275ZNONCES 0 1 2 DIFF
6892025-01-28T22:01:18.275Z------ ------------------------ ------------------------ ------------------------ -----
6902025-01-28T22:01:18.275Z 0 ed3f64193be23511302fd6d6 ed3f64193be23511302fd6d6 ed3f64193be23511302fd6d6
6912025-01-28T22:01:18.275Z
6922025-01-28T22:01:18.275Z TAGS 0 1 2 DIFF
6932025-01-28T22:01:18.275Z------ -------------------------------- -------------------------------- -------------------------------- -----
6942025-01-28T22:01:18.275Z 0 494045e0eca68c118328fa939a76a85a 494045e0eca68c118328fa939a76a85a 494045e0eca68c118328fa939a76a85a
6952025-01-28T22:01:18.276Z
6962025-01-28T22:01:18.276ZHASHES 0 1 2 DIFF
6972025-01-28T22:01:18.276Z------ ---------------- ---------------- ---------------- -----
6982025-01-28T22:01:18.276Z
6992025-01-28T22:01:18.278Zdump block test passed
7002025-01-28T22:01:18.279ZInitial upstairs tests have completed, stopping all downstairs
7012025-01-28T22:01:23.349ZCreating 7 larger downstairs regions
7022025-01-28T22:01:23.527ZStarting 7 downstairs
7032025-01-28T22:01:28.537Zdsc restarted at PID: 1260
7042025-01-28T22:01:28.922ZNow do the replace-reconcile test
7052025-01-28T22:01:28.979ZUsing 8870 for the replacement port
7062025-01-28T22:04:25.425ZCompleted test: replace-reconcile
7072025-01-28T22:04:25.426ZNow do the replace-before-active test
7082025-01-28T22:07:59.487ZCompleted test: replace-before-active
7092025-01-28T22:07:59.490ZAll tests have completed, stopping all downstairs
7102025-01-28T22:07:59.546Z
7112025-01-28T22:07:59.549ZAll Tests have passed
7122025-01-28T22:07:59.549Z13:35 Test duration
7132025-01-28T22:07:59.549Z
7142025-01-28T22:07:59.549Zreal 13:35.393957079
7152025-01-28T22:07:59.549Zuser 18:32.735206373
7162025-01-28T22:07:59.549Zsys 24.460637666
7172025-01-28T22:07:59.549Ztrap 0.526142483
7182025-01-28T22:07:59.549Ztflt 0.045756729
7192025-01-28T22:07:59.549Zdflt 0.034312691
7202025-01-28T22:07:59.549Zkflt 0.001424528
7212025-01-28T22:07:59.549Zlock 25:18:09.176619827
7222025-01-28T22:07:59.549Zslp 1:14:03.869347654
7232025-01-28T22:07:59.549Zlat 2:00.213185775
7242025-01-28T22:07:59.549Zstop 0.216085522
7252025-01-28T22:07:59.549Z+ echo 'test-up-2region-encrypted ends'
7262025-01-28T22:07:59.550Ztest-up-2region-encrypted ends
7272025-01-28T22:08:04.556Zprocess exited: duration 823772 ms, exit code 0
7282025-01-28T22:08:04.556Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
7292025-01-28T22:09:04.628Zexec warning: : stdout descriptor may be held open by a background process; giving up!
7302025-01-28T22:09:04.628Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
7312025-01-28T22:09:04.639Zfound 17 output files
7322025-01-28T22:09:04.647Zuploading: /tmp/test_up-build/dsc-out.txt (50722 bytes)
7332025-01-28T22:09:05.654Zuploaded: /tmp/test_up-build/dsc-out.txt
7342025-01-28T22:09:05.664Zuploading: /tmp/test_up-build/test_up_out.txt (7883552 bytes)
7352025-01-28T22:09:06.815Zuploaded: /tmp/test_up-build/test_up_out.txt
7362025-01-28T22:09:06.815Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (147257 bytes)
7372025-01-28T22:09:07.834Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
7382025-01-28T22:09:07.837Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (144639 bytes)
7392025-01-28T22:09:08.854Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
7402025-01-28T22:09:08.857Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (251031 bytes)
7412025-01-28T22:09:09.879Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
7422025-01-28T22:09:09.882Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (139418 bytes)
7432025-01-28T22:09:10.900Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
7442025-01-28T22:09:10.903Zuploading: /tmp/test_up-build/dsc/downstairs-8850.txt (358123 bytes)
7452025-01-28T22:09:11.928Zuploaded: /tmp/test_up-build/dsc/downstairs-8850.txt
7462025-01-28T22:09:11.929Zuploading: /tmp/test_up-build/dsc/downstairs-8860.txt (33795 bytes)
7472025-01-28T22:09:12.941Zuploaded: /tmp/test_up-build/dsc/downstairs-8860.txt
7482025-01-28T22:09:12.941Zuploading: /tmp/test_up-build/dsc/downstairs-8870.txt (730543 bytes)
7492025-01-28T22:09:13.982Zuploaded: /tmp/test_up-build/dsc/downstairs-8870.txt
7502025-01-28T22:09:13.982Zuploading: /tmp/debug/df.txt (1270 bytes)
7512025-01-28T22:09:14.991Zuploaded: /tmp/debug/df.txt
7522025-01-28T22:09:14.991Zuploading: /tmp/debug/dtrace.txt (980918 bytes)
7532025-01-28T22:09:16.035Zuploaded: /tmp/debug/dtrace.txt
7542025-01-28T22:09:16.039Zuploading: /tmp/debug/iostat.txt (264367 bytes)
7552025-01-28T22:09:16.058Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 264367 -> 267667
7562025-01-28T22:09:17.066Zuploaded: /tmp/debug/iostat.txt
7572025-01-28T22:09:17.066Zuploading: /tmp/debug/mpstat.txt (642259 bytes)
7582025-01-28T22:09:17.101Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 642259 -> 651007
7592025-01-28T22:09:18.107Zuploaded: /tmp/debug/mpstat.txt
7602025-01-28T22:09:18.107Zuploading: /tmp/debug/paging.txt (115377 bytes)
7612025-01-28T22:09:18.120Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 115377 -> 117030
7622025-01-28T22:09:19.128Zuploaded: /tmp/debug/paging.txt
7632025-01-28T22:09:19.132Zuploading: /tmp/debug/prstat.txt (1253508 bytes)
7642025-01-28T22:09:19.177Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 1253508 -> 1272883
7652025-01-28T22:09:20.188Zuploaded: /tmp/debug/prstat.txt
7662025-01-28T22:09:20.191Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
7672025-01-28T22:09:21.197Zuploaded: /tmp/debug/psrinfo.txt
7682025-01-28T22:09:21.197Zuploading: /tmp/debug/upstairs-info.txt (180510 bytes)
7692025-01-28T22:09:22.219Zuploaded: /tmp/debug/upstairs-info.txt