01JMYZ2K8M1ZJAV62E96E9KJ7N: test-up-2region-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01JMYZ6S8S4E26C1GQP2KAD60Y

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12025-02-25T16:45:50.884Zjob dependencies complete; ready to run (waiting for 13 m 54 s)
22025-02-25T17:18:13.544Zjob assigned to worker 01JMZ1T1KT26PCV0F4D7AXFT2S [factory aws, i-0acef534079d10197] (queued for 32 m 22 s)
32025-02-25T17:18:22.596Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42025-02-25T17:18:23.803Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52025-02-25T17:18:23.804Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62025-02-25T17:18:24.855Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72025-02-25T17:18:24.856Zdownloading input: /input/build/work/bins/crutest.gz
82025-02-25T17:18:26.020Zdownloaded input: /input/build/work/bins/crutest.gz
92025-02-25T17:18:26.020Zdownloading input: /input/build/work/bins/dsc.gz
102025-02-25T17:18:26.708Zdownloaded input: /input/build/work/bins/dsc.gz
112025-02-25T17:18:26.711Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122025-02-25T17:18:26.729Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132025-02-25T17:18:26.729Zdownloading input: /input/build/work/scripts/test_ds.sh
142025-02-25T17:18:26.759Zdownloaded input: /input/build/work/scripts/test_ds.sh
152025-02-25T17:18:26.759Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162025-02-25T17:18:26.777Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172025-02-25T17:18:26.777Zdownloading input: /input/build/work/scripts/test_repair.sh
182025-02-25T17:18:26.953Zdownloaded input: /input/build/work/scripts/test_repair.sh
192025-02-25T17:18:26.953Zdownloading input: /input/build/work/scripts/test_replay.sh
202025-02-25T17:18:26.969Zdownloaded input: /input/build/work/scripts/test_replay.sh
212025-02-25T17:18:26.969Zdownloading input: /input/build/work/scripts/test_up.sh
222025-02-25T17:18:26.989Zdownloaded input: /input/build/work/scripts/test_up.sh
232025-02-25T17:18:26.989Zdownloading input: /input/build/work/scripts/upstairs_info.d
242025-02-25T17:18:27.021Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252025-02-25T17:18:27.023Zdownloading input: /input/build/tmp/cargo-test-out.log
262025-02-25T17:18:27.742Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272025-02-25T17:18:27.742Zstarting task 0: "setup"
282025-02-25T17:18:27.747Z++ uname -s
292025-02-25T17:18:27.750Z+ kern=SunOS
302025-02-25T17:18:27.750Z+ build_user=build
312025-02-25T17:18:27.750Z+ build_uid=12345
322025-02-25T17:18:27.750Z+ work_dir=/work
332025-02-25T17:18:27.750Z+ input_dir=/input
342025-02-25T17:18:27.750Z+ [[ 0 == 12345 ]]
352025-02-25T17:18:27.750Z+ case "$kern" in
362025-02-25T17:18:27.750Z+ groupadd -g 12345 build
372025-02-25T17:18:27.765Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382025-02-25T17:18:29.761Z+ zfs create -o mountpoint=/work rpool/work
392025-02-25T17:18:30.245Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402025-02-25T17:18:30.248Z+ home_fs=zfs
412025-02-25T17:18:30.248Z+ [[ zfs == autofs ]]
422025-02-25T17:18:30.248Z+ mkdir -p /home/build
432025-02-25T17:18:30.253Z+ chown build:build /home/build /work
442025-02-25T17:18:31.251Z+ chmod 0700 /home/build /work
452025-02-25T17:18:31.259Zprocess exited: duration 3514 ms, exit code 0
 
462025-02-25T17:18:31.266Zstarting task 1: "authentication"
472025-02-25T17:18:31.283Zprocess exited: duration 17 ms, exit code 0
 
482025-02-25T17:18:31.293Zstarting task 2: "build"
492025-02-25T17:18:31.297Z+ banner cores
502025-02-25T17:18:31.316Z
512025-02-25T17:18:31.316Z #### #### ##### ###### ####
522025-02-25T17:18:31.316Z # # # # # # # #
532025-02-25T17:18:31.317Z # # # # # ##### ####
542025-02-25T17:18:31.317Z # # # ##### # #
552025-02-25T17:18:31.317Z # # # # # # # # #
562025-02-25T17:18:31.317Z #### #### # # ###### ####
572025-02-25T17:18:31.317Z
582025-02-25T17:18:31.321Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
592025-02-25T17:18:31.324Z+ banner unpack
602025-02-25T17:18:31.328Z
612025-02-25T17:18:31.328Z # # # # ##### ## #### # #
622025-02-25T17:18:31.328Z # # ## # # # # # # # # #
632025-02-25T17:18:31.328Z # # # # # # # # # # ####
642025-02-25T17:18:31.328Z # # # # # ##### ###### # # #
652025-02-25T17:18:31.328Z # # # ## # # # # # # #
662025-02-25T17:18:31.328Z #### # # # # # #### # #
672025-02-25T17:18:31.328Z
682025-02-25T17:18:31.328Z+ mkdir -p /var/tmp/bins
692025-02-25T17:18:31.332Z+ for t in "$input/bins/"*.gz
702025-02-25T17:18:31.332Z++ basename /input/build/work/bins/crucible-downstairs.gz
712025-02-25T17:18:31.332Z+ b=crucible-downstairs.gz
722025-02-25T17:18:31.332Z+ b=crucible-downstairs
732025-02-25T17:18:31.332Z+ gunzip
742025-02-25T17:18:33.736Z+ chmod +x /var/tmp/bins/crucible-downstairs
752025-02-25T17:18:33.740Z+ for t in "$input/bins/"*.gz
762025-02-25T17:18:33.740Z++ basename /input/build/work/bins/crucible-hammer.gz
772025-02-25T17:18:33.743Z+ b=crucible-hammer.gz
782025-02-25T17:18:33.743Z+ b=crucible-hammer
792025-02-25T17:18:33.743Z+ gunzip
802025-02-25T17:18:35.928Z+ chmod +x /var/tmp/bins/crucible-hammer
812025-02-25T17:18:35.932Z+ for t in "$input/bins/"*.gz
822025-02-25T17:18:35.932Z++ basename /input/build/work/bins/crutest.gz
832025-02-25T17:18:35.932Z+ b=crutest.gz
842025-02-25T17:18:35.932Z+ b=crutest
852025-02-25T17:18:35.932Z+ gunzip
862025-02-25T17:18:38.305Z+ chmod +x /var/tmp/bins/crutest
872025-02-25T17:18:38.308Z+ for t in "$input/bins/"*.gz
882025-02-25T17:18:38.310Z++ basename /input/build/work/bins/dsc.gz
892025-02-25T17:18:38.310Z+ b=dsc.gz
902025-02-25T17:18:38.310Z+ b=dsc
912025-02-25T17:18:38.310Z+ gunzip
922025-02-25T17:18:39.718Z+ chmod +x /var/tmp/bins/dsc
932025-02-25T17:18:39.721Z+ export BINDIR=/var/tmp/bins
942025-02-25T17:18:39.721Z+ BINDIR=/var/tmp/bins
952025-02-25T17:18:39.721Z+ jobpid=1147
962025-02-25T17:18:39.724Z+ echo 'Setup debug logging'
972025-02-25T17:18:39.724ZSetup debug logging
982025-02-25T17:18:39.724Z+ mkdir /tmp/debug
992025-02-25T17:18:39.724Z+ sleep 3600
1002025-02-25T17:18:39.724Z+ psrinfo -v
1012025-02-25T17:18:39.727Z+ df -h
1022025-02-25T17:18:39.751Z+ prstat -d d -mLc 1
1032025-02-25T17:18:39.754Z+ iostat -T d -xn 1
1042025-02-25T17:18:39.754Z+ mpstat -T d 1
1052025-02-25T17:18:39.754Z+ vmstat -T d -p 1
1062025-02-25T17:18:39.757Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1072025-02-25T17:18:39.757Z+ banner test_up_2r_encrypted
1082025-02-25T17:18:39.757Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1092025-02-25T17:18:39.757Z #####
1102025-02-25T17:18:39.757Z ##### ###### #### ##### # # ##### # # #####
1112025-02-25T17:18:39.757Z # # # # # # # # # # #
1122025-02-25T17:18:39.757Z # ##### #### # # # # # ##### # #
1132025-02-25T17:18:39.757Z # # # # # # ##### # #####
1142025-02-25T17:18:39.757Z # # # # # # # # # # #
1152025-02-25T17:18:39.757Z # ###### #### # ####### #### # ####### ####### # #
1162025-02-25T17:18:39.757Z
1172025-02-25T17:18:39.757Z+ ptime -m bash /input/build/work/scripts/test_up.sh -r 2 -N encrypted
1182025-02-25T17:18:39.768Z/input/build/work
1192025-02-25T17:18:39.796ZUsing 2 region sets
1202025-02-25T17:18:39.796ZTurn off color for downstairs dump
1212025-02-25T17:18:39.830ZUpstairs using key: w5TsebgG58kGiw98rGR+3jLHeMw3HrAuB2/Xc5e1mRo=
1222025-02-25T17:18:39.833Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1232025-02-25T17:18:39.833ZCreating 6 downstairs regions
1242025-02-25T17:18:40.159ZStarting 6 downstairs
1252025-02-25T17:18:40.161Zdsc started at PID: 1195
1262025-02-25T17:18:45.515ZDisable automatic restart on all downstairs
1272025-02-25T17:18:45.570Z
1282025-02-25T17:18:45.570ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1292025-02-25T17:18:45.574ZRunning test: span
1302025-02-25T17:18:49.712ZCompleted test: span
1312025-02-25T17:18:54.720ZRunning test: big
1322025-02-25T17:18:59.320ZCompleted test: big
1332025-02-25T17:19:04.311ZRunning test: dep
1342025-02-25T17:19:38.042ZCompleted test: dep
1352025-02-25T17:19:43.049ZRunning test: balloon
1362025-02-25T17:23:57.541ZCompleted test: balloon
1372025-02-25T17:24:02.548ZRunning test: deactivate
1382025-02-25T17:24:57.268ZCompleted test: deactivate
1392025-02-25T17:25:02.276ZRunning hammer
1402025-02-25T17:25:14.827ZRun repair tests
1412025-02-25T17:25:14.831Z/var/tmp/bins/crutest fill -g 31 -q --key w5TsebgG58kGiw98rGR+3jLHeMw3HrAuB2/Xc5e1mRo= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1422025-02-25T17:25:14.891Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.689082312Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"task":"crutest"}
1432025-02-25T17:25:14.894Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.689601155Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"task":"crutest"}
1442025-02-25T17:25:14.894Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.691201459Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"task":"crutest"}
1452025-02-25T17:25:14.894Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.691383205Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238}
1462025-02-25T17:25:14.894Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"47aa4c863f974975b86a01cc6893b0f87d94aae1\",\n git_commit_timestamp: \"2025-02-25T16:27:18.000000000Z\",\n git_branch: \"renovate/thiserror-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.691437578Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238}
1472025-02-25T17:25:14.894Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.691466085Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238}
1482025-02-25T17:25:14.894Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.691484592Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238}
1492025-02-25T17:25:14.897Z{"msg":"Crucible 4da62470-d807-4bb4-9e5c-259a15fc6915 has session id: aa802e9b-93e5-4821-96f7-0f6acda64bf0","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.692183152Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1502025-02-25T17:25:14.897Z{"msg":"Upstairs opts: Upstairs UUID: 4da62470-d807-4bb4-9e5c-259a15fc6915, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.692256263Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1512025-02-25T17:25:14.897Z{"msg":"Crucible stats registered with UUID: 4da62470-d807-4bb4-9e5c-259a15fc6915","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.692306136Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1522025-02-25T17:25:14.897Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:25:13.692329803Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1532025-02-25T17:25:14.897Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.694279642Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"task":"crutest"}
1542025-02-25T17:25:14.897Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.69436661Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238}
1552025-02-25T17:25:14.897Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"47aa4c863f974975b86a01cc6893b0f87d94aae1\",\n git_commit_timestamp: \"2025-02-25T16:27:18.000000000Z\",\n git_branch: \"renovate/thiserror-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.694403426Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238}
1562025-02-25T17:25:14.897Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.694433302Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238}
1572025-02-25T17:25:14.897Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.694456379Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238}
1582025-02-25T17:25:14.897Z{"msg":"Crucible 4da62470-d807-4bb4-9e5c-259a15fc6915 has session id: 3f180d9e-dce8-463a-9adc-1ad11a6ee2b1","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.694884424Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
1592025-02-25T17:25:14.897Z{"msg":"Upstairs opts: Upstairs UUID: 4da62470-d807-4bb4-9e5c-259a15fc6915, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.694991699Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
1602025-02-25T17:25:14.897Z{"msg":"Crucible stats registered with UUID: 4da62470-d807-4bb4-9e5c-259a15fc6915","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.695024395Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
1612025-02-25T17:25:14.897Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:25:13.695060391Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
1622025-02-25T17:25:14.900Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.695660523Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"task":"crutest"}
1632025-02-25T17:25:14.900Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.695703688Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238}
1642025-02-25T17:25:14.900Z{"msg":"4da62470-d807-4bb4-9e5c-259a15fc6915 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.695802305Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1652025-02-25T17:25:14.900Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.695827712Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"client":"0","":"downstairs","session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1662025-02-25T17:25:14.900Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.695855648Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"client":"0","":"downstairs","session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1672025-02-25T17:25:14.900Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.695879035Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"client":"1","":"downstairs","session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1682025-02-25T17:25:14.901Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.695922959Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"client":"1","":"downstairs","session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1692025-02-25T17:25:14.901Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.695966894Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"client":"2","":"downstairs","session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1702025-02-25T17:25:14.901Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.69599873Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"client":"2","":"downstairs","session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1712025-02-25T17:25:14.901Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.696034755Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1722025-02-25T17:25:14.901Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.696166268Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1732025-02-25T17:25:14.901Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.696297171Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1742025-02-25T17:25:14.901Z{"msg":"ds_connection connected from Ok(127.0.0.1:49507)","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.696342285Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1752025-02-25T17:25:14.901Z{"msg":"ds_connection connected from Ok(127.0.0.1:48856)","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.69638584Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1762025-02-25T17:25:14.901Z{"msg":"ds_connection connected from Ok(127.0.0.1:46434)","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.696427114Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1772025-02-25T17:25:14.901Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.696891654Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"client":"1","":"downstairs","session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1782025-02-25T17:25:14.901Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.696932539Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"client":"0","":"downstairs","session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1792025-02-25T17:25:14.901Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.696960136Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"client":"2","":"downstairs","session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1802025-02-25T17:25:14.901Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.697582555Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"client":"1","":"downstairs","session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1812025-02-25T17:25:14.901Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.69762412Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"client":"0","":"downstairs","session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1822025-02-25T17:25:14.901Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.697651807Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"client":"2","":"downstairs","session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1832025-02-25T17:25:14.901Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.69801205Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1842025-02-25T17:25:14.901Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.698046645Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1852025-02-25T17:25:14.901Z{"msg":"[0]R flush_numbers: [3995, 3995, 3995, 3995, 3995]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.698060834Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"downstairs","":"downstairs","session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1862025-02-25T17:25:14.901Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.698082611Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"downstairs","":"downstairs","session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1872025-02-25T17:25:14.901Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.698096429Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"downstairs","":"downstairs","session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1882025-02-25T17:25:14.901Z{"msg":"[1]R flush_numbers: [3995, 3995, 3995, 3995, 3995]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.698117216Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"downstairs","":"downstairs","session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1892025-02-25T17:25:14.901Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.698130395Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"downstairs","":"downstairs","session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1902025-02-25T17:25:14.901Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.698143443Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"downstairs","":"downstairs","session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1912025-02-25T17:25:14.901Z{"msg":"[2]R flush_numbers: [3995, 3995, 3995, 3995, 3995]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.69816297Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"downstairs","":"downstairs","session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1922025-02-25T17:25:14.901Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.698180358Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"downstairs","":"downstairs","session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1932025-02-25T17:25:14.901Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.698214364Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"downstairs","":"downstairs","session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1942025-02-25T17:25:14.901Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.69823984Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"downstairs","":"downstairs","session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1952025-02-25T17:25:14.901Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.698278695Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"downstairs","":"downstairs","session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1962025-02-25T17:25:14.901Z{"msg":"Next flush: 3996","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.69832197Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"downstairs","":"downstairs","session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1972025-02-25T17:25:14.905Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.698347967Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"downstairs","":"downstairs","session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1982025-02-25T17:25:14.905Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.698522564Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
1992025-02-25T17:25:14.905Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.69855672Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"client":"0","":"downstairs","session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
2002025-02-25T17:25:14.905Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.698594945Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"client":"1","":"downstairs","session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
2012025-02-25T17:25:14.905Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.698642179Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"client":"2","":"downstairs","session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
2022025-02-25T17:25:14.905Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.698753174Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
2032025-02-25T17:25:14.905Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.698781761Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
2042025-02-25T17:25:14.905Z{"msg":"4da62470-d807-4bb4-9e5c-259a15fc6915 is now active with session: aa802e9b-93e5-4821-96f7-0f6acda64bf0","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.698819956Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
2052025-02-25T17:25:14.905Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.698868869Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"session_id":"aa802e9b-93e5-4821-96f7-0f6acda64bf0"}
2062025-02-25T17:25:14.905Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.698909614Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238}
2072025-02-25T17:25:14.905Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.698933881Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238}
2082025-02-25T17:25:14.905Z{"msg":"4da62470-d807-4bb4-9e5c-259a15fc6915 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.698971256Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2092025-02-25T17:25:14.905Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.698997693Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"client":"0","":"downstairs","session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2102025-02-25T17:25:14.905Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.699014071Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"client":"0","":"downstairs","session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2112025-02-25T17:25:14.905Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.699037058Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"client":"1","":"downstairs","session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2122025-02-25T17:25:14.905Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.699051546Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"client":"1","":"downstairs","session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2132025-02-25T17:25:14.905Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.699077163Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"client":"2","":"downstairs","session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2142025-02-25T17:25:14.905Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.69909593Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"client":"2","":"downstairs","session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2152025-02-25T17:25:14.905Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.699119097Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2162025-02-25T17:25:14.905Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.699143404Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2172025-02-25T17:25:14.905Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.699183489Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2182025-02-25T17:25:14.905Z{"msg":"ds_connection connected from Ok(127.0.0.1:62726)","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.699215205Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2192025-02-25T17:25:14.905Z{"msg":"ds_connection connected from Ok(127.0.0.1:53047)","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.699243331Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2202025-02-25T17:25:14.905Z{"msg":"ds_connection connected from Ok(127.0.0.1:56383)","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.699333059Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2212025-02-25T17:25:14.905Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.699380903Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"client":"0","":"downstairs","session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2222025-02-25T17:25:14.905Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.699426878Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"client":"2","":"downstairs","session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2232025-02-25T17:25:14.905Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.699466062Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"client":"1","":"downstairs","session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2242025-02-25T17:25:14.905Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.699507867Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"client":"1","":"downstairs","session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2252025-02-25T17:25:14.906Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.699597785Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"client":"0","":"downstairs","session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2262025-02-25T17:25:14.906Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.699657018Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"client":"2","":"downstairs","session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2272025-02-25T17:25:14.906Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.699739857Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2282025-02-25T17:25:14.906Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.699880969Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2292025-02-25T17:25:14.906Z{"msg":"[0]R flush_numbers: [5317, 5318, 5318, 5318, 5318]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.699962978Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"downstairs","":"downstairs","session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2302025-02-25T17:25:14.906Z{"msg":"[0]R generation: [14, 14, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.699995324Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"downstairs","":"downstairs","session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2312025-02-25T17:25:14.906Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.700010762Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"downstairs","":"downstairs","session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2322025-02-25T17:25:14.906Z{"msg":"[1]R flush_numbers: [5317, 5318, 5318, 5318, 5318]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.70002457Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"downstairs","":"downstairs","session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2332025-02-25T17:25:14.906Z{"msg":"[1]R generation: [14, 14, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.700045907Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"downstairs","":"downstairs","session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2342025-02-25T17:25:14.906Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.700059816Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"downstairs","":"downstairs","session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2352025-02-25T17:25:14.906Z{"msg":"[2]R flush_numbers: [5317, 5318, 5318, 5318, 5318]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.700074064Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"downstairs","":"downstairs","session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2362025-02-25T17:25:14.906Z{"msg":"[2]R generation: [14, 14, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.700087162Z","hostname":"Wait for a query_work_queue command to finish before sending IO
2372025-02-25T17:25:14.906Zw-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"downstairs","":"downstairs","session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2382025-02-25T17:25:14.906Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.700108739Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"downstairs","":"downstairs","session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2392025-02-25T17:25:14.906Z{"msg":"Max found gen is 15","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.700121928Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"downstairs","":"downstairs","session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2402025-02-25T17:25:14.906Z{"msg":"Generation requested: 31 >= found:15","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.700134766Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"downstairs","":"downstairs","session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2412025-02-25T17:25:14.906Z{"msg":"Next flush: 5319","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.700154513Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"downstairs","":"downstairs","session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2422025-02-25T17:25:14.906Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.700167462Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"":"downstairs","":"downstairs","session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2432025-02-25T17:25:14.906Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.700192359Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2442025-02-25T17:25:14.906Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.700205357Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"client":"0","":"downstairs","session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2452025-02-25T17:25:14.906Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.700218145Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"client":"1","":"downstairs","session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2462025-02-25T17:25:14.906Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.700243072Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"client":"2","":"downstairs","session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2472025-02-25T17:25:14.906Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.70025681Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2482025-02-25T17:25:14.906Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.700269509Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2492025-02-25T17:25:14.906Z{"msg":"4da62470-d807-4bb4-9e5c-259a15fc6915 is now active with session: 3f180d9e-dce8-463a-9adc-1ad11a6ee2b1","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.700288896Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2502025-02-25T17:25:14.906Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.700313753Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238,"session_id":"3f180d9e-dce8-463a-9adc-1ad11a6ee2b1"}
2512025-02-25T17:25:14.906Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:13.700326281Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238}
2522025-02-25T17:25:14.906ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
2532025-02-25T17:25:14.906ZFill test
2542025-02-25T17:25:14.955ZRead and Verify all blocks (0..100 range:false)
2552025-02-25T17:25:15.034ZWrote out file "/var/tmp/test_up-build/verify_file"
2562025-02-25T17:25:15.034ZCLIENT: Tests done. All submitted work has been ACK'd
2572025-02-25T17:25:15.034Z----------------------------------------------------------------
2582025-02-25T17:25:15.037Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2592025-02-25T17:25:15.037ZStates: Active Active Active
2602025-02-25T17:25:15.037ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2612025-02-25T17:25:15.037Z 1002 Acked Read 50 Done Done Done false
2622025-02-25T17:25:15.037Z STATES DS:0 DS:1 DS:2 TOTAL
2632025-02-25T17:25:15.037Z Sent 0 0 0 0
2642025-02-25T17:25:15.037Z Done 1 1 1 3
2652025-02-25T17:25:15.038Z Skipped 0 0 0 0
2662025-02-25T17:25:15.038Z Error 0 0 0 0
2672025-02-25T17:25:15.038ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2682025-02-25T17:25:15.038ZDownstairs last five completed: 1001 1000
2692025-02-25T17:25:15.038ZUpstairs last five completed: 1002 1001 1000
2702025-02-25T17:25:15.038Z----------------------------------------------------------------
2712025-02-25T17:25:15.038Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2722025-02-25T17:25:15.038ZStates: Active Active Active
2732025-02-25T17:25:15.038ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2742025-02-25T17:25:15.038Z 1002 Acked Read 50 Done Done Done false
2752025-02-25T17:25:15.038Z STATES DS:0 DS:1 DS:2 TOTAL
2762025-02-25T17:25:15.038Z Sent 0 0 0 0
2772025-02-25T17:25:15.038Z Done 1 1 1 3
2782025-02-25T17:25:15.038Z Skipped 0 0 0 0
2792025-02-25T17:25:15.038Z Error 0 0 0 0
2802025-02-25T17:25:15.038ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2812025-02-25T17:25:15.038ZDownstairs last five completed: 1001 1000
2822025-02-25T17:25:15.038ZUpstairs last five completed: 1002 1001 1000
2832025-02-25T17:25:15.038ZCLIENT: Up:0 ds:2 act:6
2842025-02-25T17:25:19.036Z----------------------------------------------------------------
2852025-02-25T17:25:19.036Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2862025-02-25T17:25:19.036ZDownstairs last five completed: 1003 1002 1001 1000
2872025-02-25T17:25:19.036ZUpstairs last five completed: 1003 1002 1001 1000
2882025-02-25T17:25:19.041Z----------------------------------------------------------------
2892025-02-25T17:25:19.041Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2902025-02-25T17:25:19.041ZDownstairs last five completed: 1003 1002 1001 1000
2912025-02-25T17:25:19.042ZUpstairs last five completed: 1003 1002 1001 1000
2922025-02-25T17:25:19.042ZCLIENT: Up:0 ds:0 act:6
2932025-02-25T17:25:19.042ZCLIENT: All crucible jobs finished, exiting program
2942025-02-25T17:25:19.042Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:25:17.832866832Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1238}
2952025-02-25T17:25:19.047ZRepair setup passed
2962025-02-25T17:25:19.052ZCopy the region for /var/tmp/test_up-build/8830
2972025-02-25T17:25:19.052Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2982025-02-25T17:25:19.055Z/var/tmp/bins/crutest fill -g 32 -q --key w5TsebgG58kGiw98rGR+3jLHeMw3HrAuB2/Xc5e1mRo= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2992025-02-25T17:25:19.112Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.909520374Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"task":"crutest"}
3002025-02-25T17:25:19.116Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.91008783Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"task":"crutest"}
3012025-02-25T17:25:19.116Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.911703712Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"task":"crutest"}
3022025-02-25T17:25:19.116Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.911878669Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244}
3032025-02-25T17:25:19.116Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"47aa4c863f974975b86a01cc6893b0f87d94aae1\",\n git_commit_timestamp: \"2025-02-25T16:27:18.000000000Z\",\n git_branch: \"renovate/thiserror-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.911931032Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244}
3042025-02-25T17:25:19.116Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.91195458Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244}
3052025-02-25T17:25:19.116Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.911970107Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244}
3062025-02-25T17:25:19.119Z{"msg":"Crucible 2cd2eca7-9bd0-4e31-b07b-b4d11700af20 has session id: d2effda4-a22c-4de3-99a0-ac22925a6363","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.912531265Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3072025-02-25T17:25:19.119Z{"msg":"Upstairs opts: Upstairs UUID: 2cd2eca7-9bd0-4e31-b07b-b4d11700af20, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.91257142Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3082025-02-25T17:25:19.119Z{"msg":"Crucible stats registered with UUID: 2cd2eca7-9bd0-4e31-b07b-b4d11700af20","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.912586978Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3092025-02-25T17:25:19.119Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:25:17.912600246Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3102025-02-25T17:25:19.119Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.914561083Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"task":"crutest"}
3112025-02-25T17:25:19.119Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.914625225Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244}
3122025-02-25T17:25:19.119Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"47aa4c863f974975b86a01cc6893b0f87d94aae1\",\n git_commit_timestamp: \"2025-02-25T16:27:18.000000000Z\",\n git_branch: \"renovate/thiserror-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.91466604Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244}
3132025-02-25T17:25:19.119Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.914689887Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244}
3142025-02-25T17:25:19.119Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.914707574Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244}
3152025-02-25T17:25:19.119Z{"msg":"Crucible 2cd2eca7-9bd0-4e31-b07b-b4d11700af20 has session id: 76e96f3f-cd29-4510-a142-b02465ac8967","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.91512873Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3162025-02-25T17:25:19.119Z{"msg":"Upstairs opts: Upstairs UUID: 2cd2eca7-9bd0-4e31-b07b-b4d11700af20, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.915168025Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3172025-02-25T17:25:19.119Z{"msg":"Crucible stats registered with UUID: 2cd2eca7-9bd0-4e31-b07b-b4d11700af20","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.915201041Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3182025-02-25T17:25:19.119Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:25:17.915222938Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3192025-02-25T17:25:19.119Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.915890162Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"task":"crutest"}
3202025-02-25T17:25:19.119Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.915930516Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244}
3212025-02-25T17:25:19.119Z{"msg":"2cd2eca7-9bd0-4e31-b07b-b4d11700af20 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.916017545Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3222025-02-25T17:25:19.123Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.916049221Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"client":"0","":"downstairs","session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3232025-02-25T17:25:19.123Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.916099595Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"client":"0","":"downstairs","session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3242025-02-25T17:25:19.123Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.916142769Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"client":"1","":"downstairs","session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3252025-02-25T17:25:19.123Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.916168726Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"client":"1","":"downstairs","session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3262025-02-25T17:25:19.123Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.916204591Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"client":"2","":"downstairs","session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3272025-02-25T17:25:19.123Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.916242866Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"client":"2","":"downstairs","session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3282025-02-25T17:25:19.123Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.916281721Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3292025-02-25T17:25:19.123Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.916564105Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3302025-02-25T17:25:19.123Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.91660242Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3312025-02-25T17:25:19.123Z{"msg":"ds_connection connected from Ok(127.0.0.1:42026)","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.916627337Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3322025-02-25T17:25:19.123Z{"msg":"ds_connection connected from Ok(127.0.0.1:36288)","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.916653593Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3332025-02-25T17:25:19.123Z{"msg":"ds_connection connected from Ok(127.0.0.1:41697)","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.91667863Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3342025-02-25T17:25:19.123Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.916976592Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"client":"1","":"downstairs","session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3352025-02-25T17:25:19.123Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.917048902Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"client":"2","":"downstairs","session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3362025-02-25T17:25:19.123Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.917080128Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"client":"0","":"downstairs","session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3372025-02-25T17:25:19.123Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.917617749Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"client":"1","":"downstairs","session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3382025-02-25T17:25:19.123Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.917725285Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"client":"2","":"downstairs","session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3392025-02-25T17:25:19.123Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.917768379Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"client":"0","":"downstairs","session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3402025-02-25T17:25:19.123Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.917986921Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3412025-02-25T17:25:19.123Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.918087548Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3422025-02-25T17:25:19.123Z{"msg":"[0]R flush_numbers: [3996, 3996, 3996, 3996, 3996]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.918121554Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"downstairs","":"downstairs","session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3432025-02-25T17:25:19.123Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.918140841Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"downstairs","":"downstairs","session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3442025-02-25T17:25:19.123Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.918171018Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"downstairs","":"downstairs","session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3452025-02-25T17:25:19.123Z{"msg":"[1]R flush_numbers: [3996, 3996, 3996, 3996, 3996]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.918190955Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"downstairs","":"downstairs","session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3462025-02-25T17:25:19.123Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.918218661Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"downstairs","":"downstairs","session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3472025-02-25T17:25:19.123Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.918236719Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"downstairs","":"downstairs","session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3482025-02-25T17:25:19.123Z{"msg":"[2]R flush_numbers: [3996, 3996, 3996, 3996, 3996]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.918264955Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"downstairs","":"downstairs","session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3492025-02-25T17:25:19.123Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.918282403Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"downstairs","":"downstairs","session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3502025-02-25T17:25:19.123Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.918312059Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"downstairs","":"downstairs","session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3512025-02-25T17:25:19.123Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.918328947Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"downstairs","":"downstairs","session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3522025-02-25T17:25:19.123Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.918366952Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"downstairs","":"downstairs","session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3532025-02-25T17:25:19.123Z{"msg":"Next flush: 3997","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.918390599Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"downstairs","":"downstairs","session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3542025-02-25T17:25:19.123Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.918413456Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"downstairs","":"downstairs","session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3552025-02-25T17:25:19.123Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.918436323Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3562025-02-25T17:25:19.123Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.91845667Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"client":"0","":"downstairs","session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3572025-02-25T17:25:19.123Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.918493816Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"client":"1","":"downstairs","session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3582025-02-25T17:25:19.123Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.918657765Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"client":"2","":"downstairs","session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3592025-02-25T17:25:19.124Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.918708378Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3602025-02-25T17:25:19.124Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.918735055Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3612025-02-25T17:25:19.124Z{"msg":"2cd2eca7-9bd0-4e31-b07b-b4d11700af20 is now active with session: d2effda4-a22c-4de3-99a0-ac22925a6363","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.918756992Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3622025-02-25T17:25:19.124Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.918785948Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"session_id":"d2effda4-a22c-4de3-99a0-ac22925a6363"}
3632025-02-25T17:25:19.124Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.918799326Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244}
3642025-02-25T17:25:19.124Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.918814075Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244}
3652025-02-25T17:25:19.124Z{"msg":"2cd2eca7-9bd0-4e31-b07b-b4d11700af20 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.91884725Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3662025-02-25T17:25:19.124Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.918868857Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"client":"0","":"downstairs","session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3672025-02-25T17:25:19.124Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.918902513Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"client":"0","":"downstairs","session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3682025-02-25T17:25:19.124Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.918931469Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"client":"1","":"downstairs","session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3692025-02-25T17:25:19.124Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.918957066Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"client":"1","":"downstairs","session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3702025-02-25T17:25:19.124Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.918978053Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"client":"2","":"downstairs","session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3712025-02-25T17:25:19.124Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.91900223Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"client":"2","":"downstairs","session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3722025-02-25T17:25:19.124Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.919028867Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3732025-02-25T17:25:19.124Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.919055153Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3742025-02-25T17:25:19.124Z{"msg":"ds_connection connected from Ok(127.0.0.1:41395)","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.919086099Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3752025-02-25T17:25:19.127Z{"msg":"ds_connection connected from Ok(127.0.0.1:52611)","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.919110526Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3762025-02-25T17:25:19.127Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.919135323Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3772025-02-25T17:25:19.127Z{"msg":"ds_connection connected from Ok(127.0.0.1:55503)","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.91915791Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3782025-02-25T17:25:19.127Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.919189816Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"client":"0","":"downstairs","session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3792025-02-25T17:25:19.127Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.919204864Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"client":"2","":"downstairs","session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3802025-02-25T17:25:19.127Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.919225921Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"client":"1","":"downstairs","session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3812025-02-25T17:25:19.127Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.91946705Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"client":"2","":"downstairs","session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3822025-02-25T17:25:19.127Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.919513544Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"client":"1","":"downstairs","session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3832025-02-25T17:25:19.127Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.919664185Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"client":"0","":"downstairs","session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3842025-02-25T17:25:19.127Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.919842352Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3852025-02-25T17:25:19.127Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.919867139Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3862025-02-25T17:25:19.127Z{"msg":"[0]R flush_numbers: [5319, 5319, 5319, 5319, 5319]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.920042986Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"downstairs","":"downstairs","session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3872025-02-25T17:25:19.127Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.920076222Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"downstairs","":"downstairs","session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3882025-02-25T17:25:19.127Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.920094669Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"downstairs","":"downstairs","session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3892025-02-25T17:25:19.127Z{"msg":"[1]R flush_numbers: [5319, 5319, 5319, 5319, 5319]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.920118876Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"downstairs","":"downstairs","session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3902025-02-25T17:25:19.127Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.92016387Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"downstairs","":"downstairs","session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3912025-02-25T17:25:19.127Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.920180418Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"downstairs","":"downstairs","session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3922025-02-25T17:25:19.127Z{"msg":"[2]R flush_numbers: [5319, 5319, 5319, 5319, 5319]","v":0,"name":"crucible","level"Wait for a query_work_queue command to finish before sending IO
3932025-02-25T17:25:19.127Z:30,"time":"2025-02-25T17:25:17.920219083Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"downstairs","":"downstairs","session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3942025-02-25T17:25:19.127Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.920255318Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"downstairs","":"downstairs","session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3952025-02-25T17:25:19.128Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.920278806Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"downstairs","":"downstairs","session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3962025-02-25T17:25:19.128Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.920291994Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"downstairs","":"downstairs","session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3972025-02-25T17:25:19.128Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.92032013Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"downstairs","":"downstairs","session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3982025-02-25T17:25:19.128Z{"msg":"Next flush: 5320","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.920356456Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"downstairs","":"downstairs","session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
3992025-02-25T17:25:19.128Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.920383402Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"":"downstairs","":"downstairs","session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
4002025-02-25T17:25:19.128Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.92040053Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
4012025-02-25T17:25:19.128Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.920432646Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"client":"0","":"downstairs","session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
4022025-02-25T17:25:19.128Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.920464662Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"client":"1","":"downstairs","session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
4032025-02-25T17:25:19.128Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.92047902Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"client":"2","":"downstairs","session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
4042025-02-25T17:25:19.128Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.920518485Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
4052025-02-25T17:25:19.128Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.92055279Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
4062025-02-25T17:25:19.128Z{"msg":"2cd2eca7-9bd0-4e31-b07b-b4d11700af20 is now active with session: 76e96f3f-cd29-4510-a142-b02465ac8967","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.920569648Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
4072025-02-25T17:25:19.128Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.920581516Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244,"session_id":"76e96f3f-cd29-4510-a142-b02465ac8967"}
4082025-02-25T17:25:19.128Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:17.920611273Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244}
4092025-02-25T17:25:19.128ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
4102025-02-25T17:25:19.128ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4112025-02-25T17:25:19.128ZFill test
4122025-02-25T17:25:19.165ZRead and Verify all blocks (0..100 range:false)
4132025-02-25T17:25:19.245ZWrote out file "/var/tmp/test_up-build/verify_file"
4142025-02-25T17:25:19.246ZCLIENT: Tests done. All submitted work has been ACK'd
4152025-02-25T17:25:19.249Z----------------------------------------------------------------
4162025-02-25T17:25:19.250Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4172025-02-25T17:25:19.250ZStates: Active Active Active
4182025-02-25T17:25:19.250ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4192025-02-25T17:25:19.250Z 1002 Acked Read 50 Done Done Done false
4202025-02-25T17:25:19.250Z STATES DS:0 DS:1 DS:2 TOTAL
4212025-02-25T17:25:19.250Z Sent 0 0 0 0
4222025-02-25T17:25:19.250Z Done 1 1 1 3
4232025-02-25T17:25:19.250Z Skipped 0 0 0 0
4242025-02-25T17:25:19.250Z Error 0 0 0 0
4252025-02-25T17:25:19.250ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4262025-02-25T17:25:19.250ZDownstairs last five completed: 1001 1000
4272025-02-25T17:25:19.250ZUpstairs last five completed: 1002 1001 1000
4282025-02-25T17:25:19.250Z----------------------------------------------------------------
4292025-02-25T17:25:19.250Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4302025-02-25T17:25:19.250ZStates: Active Active Active
4312025-02-25T17:25:19.250ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4322025-02-25T17:25:19.253Z 1002 Acked Read 50 Done Done Done false
4332025-02-25T17:25:19.253Z STATES DS:0 DS:1 DS:2 TOTAL
4342025-02-25T17:25:19.253Z Sent 0 0 0 0
4352025-02-25T17:25:19.253Z Done 1 1 1 3
4362025-02-25T17:25:19.253Z Skipped 0 0 0 0
4372025-02-25T17:25:19.253Z Error 0 0 0 0
4382025-02-25T17:25:19.253ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4392025-02-25T17:25:19.253ZDownstairs last five completed: 1001 1000
4402025-02-25T17:25:19.253ZUpstairs last five completed: 1002 1001 1000
4412025-02-25T17:25:19.253ZCLIENT: Up:0 ds:2 act:6
4422025-02-25T17:25:23.247Z----------------------------------------------------------------
4432025-02-25T17:25:23.247Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4442025-02-25T17:25:23.247ZDownstairs last five completed: 1003 1002 1001 1000
4452025-02-25T17:25:23.247ZUpstairs last five completed: 1003 1002 1001 1000
4462025-02-25T17:25:23.247Z----------------------------------------------------------------
4472025-02-25T17:25:23.247Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4482025-02-25T17:25:23.247ZDownstairs last five completed: 1003 1002 1001 1000
4492025-02-25T17:25:23.247ZUpstairs last five completed: 1003 1002 1001 1000
4502025-02-25T17:25:23.250ZCLIENT: Up:0 ds:0 act:6
4512025-02-25T17:25:23.250ZCLIENT: All crucible jobs finished, exiting program
4522025-02-25T17:25:23.250Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:25:22.044024871Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1244}
4532025-02-25T17:25:23.254ZRepair part 1 passed
4542025-02-25T17:25:23.257Z
4552025-02-25T17:25:23.257ZKill the current downstairs
4562025-02-25T17:25:23.423ZDownstairs 2 stopped
4572025-02-25T17:25:23.423Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
4582025-02-25T17:25:23.426ZNow put back the original so we have a mismatch
4592025-02-25T17:25:23.426Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
4602025-02-25T17:25:23.429ZRestart downstairs with old directory
4612025-02-25T17:25:23.483Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4622025-02-25T17:25:23.500ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4632025-02-25T17:25:23.504Z 0 000-009 32 32 31 3997 3997 3996 F F F <---
4642025-02-25T17:25:23.504Z 1 010-019 32 32 31 3997 3997 3996 F F F <---
4652025-02-25T17:25:23.504Z 2 020-029 32 32 31 3997 3997 3996 F F F <---
4662025-02-25T17:25:23.504Z 3 030-039 32 32 31 3997 3997 3996 F F F <---
4672025-02-25T17:25:23.504Z 4 040-049 32 32 31 3997 3997 3996 F F F <---
4682025-02-25T17:25:23.504ZMax gen: 32, Max flush: 3997
4692025-02-25T17:25:23.504ZError: Difference in extent metadata found!
4702025-02-25T17:25:23.507Zdump test found error as expected
4712025-02-25T17:25:23.507Z
4722025-02-25T17:25:23.507Z
4732025-02-25T17:25:23.507Z/var/tmp/bins/crutest verify --range -g 33 -q --key w5TsebgG58kGiw98rGR+3jLHeMw3HrAuB2/Xc5e1mRo= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
4742025-02-25T17:25:23.562Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.358955382Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"task":"crutest"}
4752025-02-25T17:25:23.565Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.359496283Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"task":"crutest"}
4762025-02-25T17:25:23.565Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.361161248Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"task":"crutest"}
4772025-02-25T17:25:23.565Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.361336275Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252}
4782025-02-25T17:25:23.565Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"47aa4c863f974975b86a01cc6893b0f87d94aae1\",\n git_commit_timestamp: \"2025-02-25T16:27:18.000000000Z\",\n git_branch: \"renovate/thiserror-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.361399167Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252}
4792025-02-25T17:25:23.565Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.361428513Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252}
4802025-02-25T17:25:23.565Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.361448841Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252}
4812025-02-25T17:25:23.566Z{"msg":"Crucible da799600-243e-4133-81aa-2240f63d4f7e has session id: 79a738b6-37e9-4163-8135-cbec6cedb19e","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.362091888Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
4822025-02-25T17:25:23.566Z{"msg":"Upstairs opts: Upstairs UUID: da799600-243e-4133-81aa-2240f63d4f7e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.362131323Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
4832025-02-25T17:25:23.566Z{"msg":"Crucible stats registered with UUID: da799600-243e-4133-81aa-2240f63d4f7e","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.36215045Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
4842025-02-25T17:25:23.566Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:25:22.362183046Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
4852025-02-25T17:25:23.569Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.364157341Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"task":"crutest"}
4862025-02-25T17:25:23.569Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.364227842Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252}
4872025-02-25T17:25:23.569Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"47aa4c863f974975b86a01cc6893b0f87d94aae1\",\n git_commit_timestamp: \"2025-02-25T16:27:18.000000000Z\",\n git_branch: \"renovate/thiserror-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.364263368Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252}
4882025-02-25T17:25:23.569Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.364283245Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252}
4892025-02-25T17:25:23.569Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.364297653Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252}
4902025-02-25T17:25:23.569Z{"msg":"Crucible da799600-243e-4133-81aa-2240f63d4f7e has session id: 30778074-4489-4c12-9e19-f3fa1eac5c9d","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.364644678Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
4912025-02-25T17:25:23.569Z{"msg":"Upstairs opts: Upstairs UUID: da799600-243e-4133-81aa-2240f63d4f7e, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.364685493Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
4922025-02-25T17:25:23.569Z{"msg":"Crucible stats registered with UUID: da799600-243e-4133-81aa-2240f63d4f7e","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.364702901Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
4932025-02-25T17:25:23.569Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:25:22.364724818Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
4942025-02-25T17:25:23.569Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.365463343Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"task":"crutest"}
4952025-02-25T17:25:23.569Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.365499978Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252}
4962025-02-25T17:25:23.569Z{"msg":"da799600-243e-4133-81aa-2240f63d4f7e active request set","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.365590226Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
4972025-02-25T17:25:23.569Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.365620782Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"0","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
4982025-02-25T17:25:23.569Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.365665297Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"0","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
4992025-02-25T17:25:23.569Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.365689364Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"1","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5002025-02-25T17:25:23.569Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.365728368Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"1","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5012025-02-25T17:25:23.572Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.365772503Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"2","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5022025-02-25T17:25:23.573Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.365805409Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"2","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5032025-02-25T17:25:23.573Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.365838344Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5042025-02-25T17:25:23.573Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.365866251Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5052025-02-25T17:25:23.573Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.365899466Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5062025-02-25T17:25:23.573Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:25:22.365928783Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5072025-02-25T17:25:23.573Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:25:22.365967658Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5082025-02-25T17:25:23.573Z{"msg":"ds_connection connected from Ok(127.0.0.1:50256)","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.366002183Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5092025-02-25T17:25:23.573Z{"msg":"ds_connection connected from Ok(127.0.0.1:60986)","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.366094181Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5102025-02-25T17:25:23.573Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:25:22.366146025Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5112025-02-25T17:25:23.573Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.366234033Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5122025-02-25T17:25:23.573Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.366593527Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"0","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5132025-02-25T17:25:23.573Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.366670117Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"1","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5142025-02-25T17:25:23.573Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.367196919Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"0","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5152025-02-25T17:25:23.573Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.367300996Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"1","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5162025-02-25T17:25:23.573Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.367528186Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5172025-02-25T17:25:23.573Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:22.367599327Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5182025-02-25T17:25:33.572Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.367429759Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5192025-02-25T17:25:33.572Z{"msg":"ds_connection connected from Ok(127.0.0.1:52557)","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.367511578Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5202025-02-25T17:25:33.577Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.368756957Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"2","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5212025-02-25T17:25:33.577Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.369330594Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"2","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5222025-02-25T17:25:33.577Z{"msg":"[0]R flush_numbers: [3997, 3997, 3997, 3997, 3997]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.369689577Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"downstairs","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5232025-02-25T17:25:33.577Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.369725992Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"downstairs","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5242025-02-25T17:25:33.577Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.36974789Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"downstairs","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5252025-02-25T17:25:33.578Z{"msg":"[1]R flush_numbers: [3997, 3997, 3997, 3997, 3997]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.369785415Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"downstairs","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5262025-02-25T17:25:33.578Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.369803882Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"downstairs","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5272025-02-25T17:25:33.578Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.369840638Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"downstairs","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5282025-02-25T17:25:33.578Z{"msg":"[2]R flush_numbers: [3996, 3996, 3996, 3996, 3996]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.369880952Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"downstairs","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5292025-02-25T17:25:33.578Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.369920758Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"downstairs","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5302025-02-25T17:25:33.578Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.369963692Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"downstairs","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5312025-02-25T17:25:33.578Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.369989458Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"downstairs","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5322025-02-25T17:25:33.578Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.370042172Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"downstairs","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5332025-02-25T17:25:33.578Z{"msg":"Next flush: 3998","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.370061099Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"downstairs","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5342025-02-25T17:25:33.578Z{"msg":"extent 0 needs reconciliation: ExtentMetadata { gen: 32, flush: 3997, dirty: false } ExtentMetadata { gen: 32, flush: 3997, dirty: false } ExtentMetadata { gen: 31, flush: 3996, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.370103454Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5352025-02-25T17:25:33.578Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 3997, dirty: false }, ExtentMetadata { gen: 32, flush: 3997, dirty: false }, ExtentMetadata { gen: 31, flush: 3996, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.370146218Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5362025-02-25T17:25:33.578Z{"msg":"found dest for source 0 for extent at index 0 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.370187763Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5372025-02-25T17:25:33.578Z{"msg":"extent 1 needs reconciliation: ExtentMetadata { gen: 32, flush: 3997, dirty: false } ExtentMetadata { gen: 32, flush: 3997, dirty: false } ExtentMetadata { gen: 31, flush: 3996, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.37021168Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5382025-02-25T17:25:33.578Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 3997, dirty: false }, ExtentMetadata { gen: 32, flush: 3997, dirty: false }, ExtentMetadata { gen: 31, flush: 3996, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.370244786Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5392025-02-25T17:25:33.578Z{"msg":"found dest for source 0 for extent at index 1 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.37029225Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5402025-02-25T17:25:33.578Z{"msg":"extent 2 needs reconciliation: ExtentMetadata { gen: 32, flush: 3997, dirty: false } ExtentMetadata { gen: 32, flush: 3997, dirty: false } ExtentMetadata { gen: 31, flush: 3996, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.370337844Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5412025-02-25T17:25:33.578Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 3997, dirty: false }, ExtentMetadata { gen: 32, flush: 3997, dirty: false }, ExtentMetadata { gen: 31, flush: 3996, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.370385847Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5422025-02-25T17:25:33.578Z{"msg":"found dest for source 0 for extent at index 2 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.370428192Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5432025-02-25T17:25:33.578Z{"msg":"extent 3 needs reconciliation: ExtentMetadata { gen: 32, flush: 3997, dirty: false } ExtentMetadata { gen: 32, flush: 3997, dirty: false } ExtentMetadata { gen: 31, flush: 3996, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.370455209Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5442025-02-25T17:25:33.578Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 3997, dirty: false }, ExtentMetadata { gen: 32, flush: 3997, dirty: false }, ExtentMetadata { gen: 31, flush: 3996, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.370487394Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5452025-02-25T17:25:33.578Z{"msg":"found dest for source 0 for extent at index 3 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.370503062Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5462025-02-25T17:25:33.578Z{"msg":"extent 4 needs reconciliation: ExtentMetadata { gen: 32, flush: 3997, dirty: false } ExtentMetadata { gen: 32, flush: 3997, dirty: false } ExtentMetadata { gen: 31, flush: 3996, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.370537518Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5472025-02-25T17:25:33.581Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 3997, dirty: false }, ExtentMetadata { gen: 32, flush: 3997, dirty: false }, ExtentMetadata { gen: 31, flush: 3996, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.370561895Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5482025-02-25T17:25:33.582Z{"msg":"found dest for source 0 for extent at index 4 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.370592291Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5492025-02-25T17:25:33.582Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.370626966Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"0","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5502025-02-25T17:25:33.582Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.370653993Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"1","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5512025-02-25T17:25:33.582Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.37067163Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"2","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5522025-02-25T17:25:33.582Z{"msg":"Full repair list: {ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.370705296Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"downstairs","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5532025-02-25T17:25:33.582Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 3998, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 3998, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 3998, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 3998, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 3998, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.370827621Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"downstairs","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5542025-02-25T17:25:33.582Z{"msg":"starting reconciliation 27d77232-e7b5-41ae-b175-7263644e5c32: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.370850158Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"downstairs","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5552025-02-25T17:25:33.582Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.370863736Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"2","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5562025-02-25T17:25:33.585Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.379384166Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"2","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5572025-02-25T17:25:33.588Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.38307835Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"2","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5582025-02-25T17:25:33.591Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.386634611Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"2","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5592025-02-25T17:25:33.596Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.39028653Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"2","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5602025-02-25T17:25:33.599Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.393397099Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"downstairs","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5612025-02-25T17:25:33.599Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.393433764Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"0","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5622025-02-25T17:25:33.599Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.393450552Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"1","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5632025-02-25T17:25:33.599Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.393472269Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"2","":"downstairs","session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5642025-02-25T17:25:33.599Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.393487267Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5652025-02-25T17:25:33.599Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.393507344Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5662025-02-25T17:25:33.599Z{"msg":"da799600-243e-4133-81aa-2240f63d4f7e is now active with session: 79a738b6-37e9-4163-8135-cbec6cedb19e","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.393520513Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"session_id":"79a738b6-37e9-4163-8135-cbec6cedb19e"}
5672025-02-25T17:25:33.599Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.39354122Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252}
5682025-02-25T17:25:33.600Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.393571316Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252}
5692025-02-25T17:25:33.600Z{"msg":"da799600-243e-4133-81aa-2240f63d4f7e active request set","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.393587814Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
5702025-02-25T17:25:33.600Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.393600083Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"0","":"downstairs","session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
5712025-02-25T17:25:33.600Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.393615441Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"0","":"downstairs","session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
5722025-02-25T17:25:33.600Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.393628629Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"1","":"downstairs","session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
5732025-02-25T17:25:33.600Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.393642217Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"1","":"downstairs","session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
5742025-02-25T17:25:33.600Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.393661305Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"2","":"downstairs","session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
5752025-02-25T17:25:33.600Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.3936961Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"2","":"downstairs","session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
5762025-02-25T17:25:33.600Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.393719127Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
5772025-02-25T17:25:33.600Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.393755783Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
5782025-02-25T17:25:33.600Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.393781349Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
5792025-02-25T17:25:33.600Z{"msg":"ds_connection connected from Ok(127.0.0.1:47418)","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.393807286Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
5802025-02-25T17:25:33.600Z{"msg":"ds_connection connected from Ok(127.0.0.1:50235)","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.393832383Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
5812025-02-25T17:25:33.600Z{"msg":"ds_connection connected from Ok(127.0.0.1:53234)","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.393859359Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
5822025-02-25T17:25:33.600Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.3941618Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"0","":"downstairs","session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
5832025-02-25T17:25:33.600Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.394197706Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"2","":"downstairs","session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
5842025-02-25T17:25:33.600Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.394222752Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"1","":"downstairs","session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
5852025-02-25T17:25:33.604Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.394768882Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"1","":"downstairs","session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
5862025-02-25T17:25:33.604Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.394817366Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"2","":"downstairs","session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
5872025-02-25T17:25:33.604Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.394855321Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"0","":"downstairs","session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
5882025-02-25T17:25:33.604Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.395115117Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
5892025-02-25T17:25:33.604Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.395144823Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
5902025-02-25T17:25:33.604Z{"msg":"[0]R flush_numbers: [5320, 5320, 5320, 5320, 5320]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.395159381Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"downstairs","":"downstairs","session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
5912025-02-25T17:25:33.604Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.395181179Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"downstairs","":"downstairs","session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
5922025-02-25T17:25:33.604Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.395195107Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"downstairs","":"downstairs","session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
5932025-02-25T17:25:33.604Z{"msg":"[1]R flush_numbers: [5320, 5320, 5320, 5320, 5320]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.395208035Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"downstairs","":"downstairs","session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
5942025-02-25T17:25:33.604Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.395228622Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"downstairs","":"downstairs","session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
5952025-02-25T17:25:33.604Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.395241921Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"downstairs","":"downstairs","session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
5962025-02-25T17:25:33.604Z{"msg":"[2]R flush_numbers: [5320, 5320, 5320, 5320, 5320]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.395254639Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"downstairs","":"downstairs","session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
5972025-02-25T17:25:33.604Z{"msg":"[2]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.395267297Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"downstairs","":"downstairs","session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
5982025-02-25T17:25:33.604Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.395280736Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"downstairs","":"downstairs","session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
5992025-02-25T17:25:33.604Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.395301023Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"downstairs","":"downstairs","session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
6002025-02-25T17:25:33.604Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.395313951Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"downstairs","":"downstairs","session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
6012025-02-25T17:25:33.604Z{"msg":"Next flush: 5321","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.39532691Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"downstairs","":"downstairs","session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
6022025-02-25T17:25:33.604Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.395339378Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"":"downstairs","":"downstairs","session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
6032025-02-25T17:25:33.604Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.395365755Z"Wait for a query_work_queue command to finish before sending IO
6042025-02-25T17:25:33.604Z,"hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
6052025-02-25T17:25:33.604Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.395384412Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"0","":"downstairs","session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
6062025-02-25T17:25:33.605Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.39540575Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"1","":"downstairs","session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
6072025-02-25T17:25:33.605Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.395418958Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"client":"2","":"downstairs","session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
6082025-02-25T17:25:33.605Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.395434356Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
6092025-02-25T17:25:33.605Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.395451304Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
6102025-02-25T17:25:33.605Z{"msg":"da799600-243e-4133-81aa-2240f63d4f7e is now active with session: 30778074-4489-4c12-9e19-f3fa1eac5c9d","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.39547581Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
6112025-02-25T17:25:33.605Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.395488399Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252,"session_id":"30778074-4489-4c12-9e19-f3fa1eac5c9d"}
6122025-02-25T17:25:33.605Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:25:32.395500847Z","hostname":"w-01JMZ1T1KT26PCV0F4D7AXFT2S","pid":1252}
6132025-02-25T17:25:33.605ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
6142025-02-25T17:25:33.605ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
6152025-02-25T17:25:33.605ZRead and Verify all blocks (0..100 range:true)
6162025-02-25T17:25:33.680ZWrote out file "/var/tmp/test_up-build/verify_file"
6172025-02-25T17:25:33.680ZVerify test completed
6182025-02-25T17:25:33.684ZWrote out file "/var/tmp/test_up-build/verify_file"
6192025-02-25T17:25:33.684ZCLIENT: Tests done. All submitted work has been ACK'd
6202025-02-25T17:25:33.684Z----------------------------------------------------------------
6212025-02-25T17:25:33.684Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6222025-02-25T17:25:33.684ZStates: Active Active Active
6232025-02-25T17:25:33.684ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6242025-02-25T17:25:33.684Z 1000 Acked Read 50 Done Done Done false
6252025-02-25T17:25:33.684Z STATES DS:0 DS:1 DS:2 TOTAL
6262025-02-25T17:25:33.684Z Sent 0 0 0 0
6272025-02-25T17:25:33.684Z Done 1 1 1 3
6282025-02-25T17:25:33.684Z Skipped 0 0 0 0
6292025-02-25T17:25:33.684Z Error 0 0 0 0
6302025-02-25T17:25:33.684ZLast Flush: None None None
6312025-02-25T17:25:33.684ZDownstairs last five completed:
6322025-02-25T17:25:33.684ZUpstairs last five completed: 1000
6332025-02-25T17:25:33.684Z----------------------------------------------------------------
6342025-02-25T17:25:33.684Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6352025-02-25T17:25:33.684ZStates: Active Active Active
6362025-02-25T17:25:33.684ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6372025-02-25T17:25:33.684Z 1000 Acked Read 50 Done Done Done false
6382025-02-25T17:25:33.684Z STATES DS:0 DS:1 DS:2 TOTAL
6392025-02-25T17:25:33.684Z Sent 0 0 0 0
6402025-02-25T17:25:33.684Z Done 1 1 1 3
6412025-02-25T17:25:33.684Z Skipped 0 0 0 0
6422025-02-25T17:25:33.684Z Error 0 0 0 0
6432025-02-25T17:25:33.684ZLast Flush: None None None
6442025-02-25T17:25:33.684ZDownstairs last five completed:
6452025-02-25T17:25:33.684ZUpstairs last five completed: 1000
6462025-02-25T17:25:33.684ZCLIENT: Up:0 ds:2 act:6
6472025-02-25T17:25:37.683Z----------------------------------------------------------------
6482025-02-25T17:25:37.683Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6492025-02-25T17:25:37.683ZDownstairs last five completed: 1001 1000
6502025-02-25T17:25:37.683ZUpstairs last five completed: 1001 1000
6512025-02-25T17:25:37.683Z----------------------------------------------------------------
6522025-02-25T17:25:37.683Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6532025-02-25T17:25:37.683ZDownstairs last five completed: 1001 1000
6542025-02-25T17:25:37.683ZUpstairs last five completed: 1001 1000
6552025-02-25T17:25:37.687ZCLIENT: Up:0 ds:0 act:6
6562025-02-25T17:25:37.687ZCLIENT: All crucible jobs finished, exiting program
6572025-02-25T17:25:37.690ZRepair part 2 passed
6582025-02-25T17:25:37.693Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
6592025-02-25T17:25:37.708ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6602025-02-25T17:25:37.711Z 0 000-009 32 32 32 3997 3997 3997 F F F
6612025-02-25T17:25:37.711Z 1 010-019 32 32 32 3997 3997 3997 F F F
6622025-02-25T17:25:37.711Z 2 020-029 32 32 32 3997 3997 3997 F F F
6632025-02-25T17:25:37.711Z 3 030-039 32 32 32 3997 3997 3997 F F F
6642025-02-25T17:25:37.711Z 4 040-049 32 32 32 3997 3997 3997 F F F
6652025-02-25T17:25:37.711ZMax gen: 32, Max flush: 3997
6662025-02-25T17:25:37.714Zdump test passed
6672025-02-25T17:25:37.714Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
6682025-02-25T17:25:37.727Z Extent 1
6692025-02-25T17:25:37.729ZGEN 32 32 32
6702025-02-25T17:25:37.729ZFLUSH_ID 3997 3997 3997
6712025-02-25T17:25:37.729ZDIRTY
6722025-02-25T17:25:37.729Z
6732025-02-25T17:25:37.729ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
6742025-02-25T17:25:37.729Z010 A A A A A A
6752025-02-25T17:25:37.732Z011 A A A A A A
6762025-02-25T17:25:37.732Z012 A A A A A A
6772025-02-25T17:25:37.735Z013 A A A A A A
6782025-02-25T17:25:37.735Z014 A A A A A A
6792025-02-25T17:25:37.738Z015 A A A A A A
6802025-02-25T17:25:37.738Z016 A A A A A A
6812025-02-25T17:25:37.741Z017 A A A A A A
6822025-02-25T17:25:37.741Z018 A A A A A A
6832025-02-25T17:25:37.744Z019 A A A A A A
6842025-02-25T17:25:37.747Zdump extent test passed
6852025-02-25T17:25:37.747Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
6862025-02-25T17:25:37.762ZExtent 2 Block in extent 0 Actual block 20
6872025-02-25T17:25:37.762Z
6882025-02-25T17:25:37.765Z DATA SHA256 VER
6892025-02-25T17:25:37.765Z------ ---------------------------------------------------------------- ---
6902025-02-25T17:25:37.765Z 0 fc64baa17f03472f64436a6d84105c19485bd9a071b4c8dcdd0396e619ad0ffd A
6912025-02-25T17:25:37.768Z 1 fc64baa17f03472f64436a6d84105c19485bd9a071b4c8dcdd0396e619ad0ffd A
6922025-02-25T17:25:37.768Z 2 fc64baa17f03472f64436a6d84105c19485bd9a071b4c8dcdd0396e619ad0ffd A
6932025-02-25T17:25:37.768Z
6942025-02-25T17:25:37.768ZNONCES 0 1 2 DIFF
6952025-02-25T17:25:37.768Z------ ------------------------ ------------------------ ------------------------ -----
6962025-02-25T17:25:37.768Z 0 a5497d4d71b2b39356f14c1b a5497d4d71b2b39356f14c1b a5497d4d71b2b39356f14c1b
6972025-02-25T17:25:37.768Z
6982025-02-25T17:25:37.768Z TAGS 0 1 2 DIFF
6992025-02-25T17:25:37.768Z------ -------------------------------- -------------------------------- -------------------------------- -----
7002025-02-25T17:25:37.768Z 0 9538dd516914bed7c7ae736cd794dff5 9538dd516914bed7c7ae736cd794dff5 9538dd516914bed7c7ae736cd794dff5
7012025-02-25T17:25:37.768Z
7022025-02-25T17:25:37.768ZHASHES 0 1 2 DIFF
7032025-02-25T17:25:37.768Z------ ---------------- ---------------- ---------------- -----
7042025-02-25T17:25:37.768Z
7052025-02-25T17:25:37.771Zdump block test passed
7062025-02-25T17:25:37.771ZInitial upstairs tests have completed, stopping all downstairs
7072025-02-25T17:25:42.837ZCreating 7 larger downstairs regions
7082025-02-25T17:25:43.023ZStarting 7 downstairs
7092025-02-25T17:25:48.031Zdsc restarted at PID: 1268
7102025-02-25T17:25:48.429ZNow do the replace-reconcile test
7112025-02-25T17:25:48.485ZUsing 8870 for the replacement port
7122025-02-25T17:29:10.448ZCompleted test: replace-reconcile
7132025-02-25T17:29:10.448ZNow do the replace-before-active test
7142025-02-25T17:32:43.995ZCompleted test: replace-before-active
7152025-02-25T17:32:43.998ZAll tests have completed, stopping all downstairs
7162025-02-25T17:32:44.055Z
7172025-02-25T17:32:44.056ZAll Tests have passed
7182025-02-25T17:32:44.059Z14:04 Test duration
7192025-02-25T17:32:44.059Z
7202025-02-25T17:32:44.059Zreal 14:04.165842069
7212025-02-25T17:32:44.059Zuser 18:46.001999609
7222025-02-25T17:32:44.059Zsys 24.690498366
7232025-02-25T17:32:44.059Ztrap 0.532932373
7242025-02-25T17:32:44.059Ztflt 0.033411841
7252025-02-25T17:32:44.059Zdflt 0.101784549
7262025-02-25T17:32:44.059Zkflt 0.001566831
7272025-02-25T17:32:44.059Zlock 25:40:52.792245142
7282025-02-25T17:32:44.059Zslp 1:15:22.824417932
7292025-02-25T17:32:44.059Zlat 2:01.548962454
7302025-02-25T17:32:44.059Zstop 0.202515573
7312025-02-25T17:32:44.059Z+ echo 'test-up-2region-encrypted ends'
7322025-02-25T17:32:44.059Ztest-up-2region-encrypted ends
7332025-02-25T17:32:49.062Zprocess exited: duration 852635 ms, exit code 0
7342025-02-25T17:32:49.062Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
7352025-02-25T17:33:49.114Zexec warning: : stdout descriptor may be held open by a background process; giving up!
7362025-02-25T17:33:49.114Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
7372025-02-25T17:33:49.125Zfound 17 output files
7382025-02-25T17:33:49.125Zuploading: /tmp/test_up-build/dsc-out.txt (54951 bytes)
7392025-02-25T17:33:50.194Zuploaded: /tmp/test_up-build/dsc-out.txt
7402025-02-25T17:33:50.194Zuploading: /tmp/test_up-build/test_up_out.txt (8824965 bytes)
7412025-02-25T17:33:51.347Zuploaded: /tmp/test_up-build/test_up_out.txt
7422025-02-25T17:33:51.349Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (147266 bytes)
7432025-02-25T17:33:52.368Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
7442025-02-25T17:33:52.370Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (144659 bytes)
7452025-02-25T17:33:53.390Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
7462025-02-25T17:33:53.395Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (251032 bytes)
7472025-02-25T17:33:54.417Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
7482025-02-25T17:33:54.417Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (139432 bytes)
7492025-02-25T17:33:55.438Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
7502025-02-25T17:33:55.440Zuploading: /tmp/test_up-build/dsc/downstairs-8850.txt (2505 bytes)
7512025-02-25T17:33:56.447Zuploaded: /tmp/test_up-build/dsc/downstairs-8850.txt
7522025-02-25T17:33:56.447Zuploading: /tmp/test_up-build/dsc/downstairs-8860.txt (39016 bytes)
7532025-02-25T17:33:57.465Zuploaded: /tmp/test_up-build/dsc/downstairs-8860.txt
7542025-02-25T17:33:57.468Zuploading: /tmp/test_up-build/dsc/downstairs-8870.txt (1065305 bytes)
7552025-02-25T17:33:58.520Zuploaded: /tmp/test_up-build/dsc/downstairs-8870.txt
7562025-02-25T17:33:58.520Zuploading: /tmp/debug/df.txt (1270 bytes)
7572025-02-25T17:33:59.529Zuploaded: /tmp/debug/df.txt
7582025-02-25T17:33:59.531Zuploading: /tmp/debug/dtrace.txt (1101377 bytes)
7592025-02-25T17:34:00.581Zuploaded: /tmp/debug/dtrace.txt
7602025-02-25T17:34:00.584Zuploading: /tmp/debug/iostat.txt (273979 bytes)
7612025-02-25T17:34:00.605Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 273979 -> 277290
7622025-02-25T17:34:01.611Zuploaded: /tmp/debug/iostat.txt
7632025-02-25T17:34:01.611Zuploading: /tmp/debug/mpstat.txt (664314 bytes)
7642025-02-25T17:34:01.648Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 664314 -> 673074
7652025-02-25T17:34:02.655Zuploaded: /tmp/debug/mpstat.txt
7662025-02-25T17:34:02.658Zuploading: /tmp/debug/paging.txt (119938 bytes)
7672025-02-25T17:34:02.670Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 119938 -> 121762
7682025-02-25T17:34:03.681Zuploaded: /tmp/debug/paging.txt
7692025-02-25T17:34:03.683Zuploading: /tmp/debug/prstat.txt (1291037 bytes)
7702025-02-25T17:34:03.733Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 1291037 -> 1310424
7712025-02-25T17:34:04.740Zuploaded: /tmp/debug/prstat.txt
7722025-02-25T17:34:04.742Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
7732025-02-25T17:34:05.748Zuploaded: /tmp/debug/psrinfo.txt
7742025-02-25T17:34:05.748Zuploading: /tmp/debug/upstairs-info.txt (187115 bytes)
7752025-02-25T17:34:06.771Zuploaded: /tmp/debug/upstairs-info.txt