01JHM5GXCYH3QVJ9SK11FHRHHM: test-up-2region-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01JHM5H9ZEJTEP05TFSTRE5ZGH

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12025-01-15T05:17:35.634Zjob dependencies complete; ready to run (waiting for 13 m 47 s)
22025-01-15T05:18:35.079Zjob assigned to worker 01JHM6AXXXDCHCBNQWF5TVW69Q [factory aws, i-0d75bcef9b1f4d745] (queued for 59 s)
32025-01-15T05:18:43.529Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42025-01-15T05:18:44.629Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52025-01-15T05:18:44.630Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62025-01-15T05:18:45.681Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72025-01-15T05:18:45.681Zdownloading input: /input/build/work/bins/crutest.gz
82025-01-15T05:18:46.816Zdownloaded input: /input/build/work/bins/crutest.gz
92025-01-15T05:18:46.816Zdownloading input: /input/build/work/bins/dsc.gz
102025-01-15T05:18:47.485Zdownloaded input: /input/build/work/bins/dsc.gz
112025-01-15T05:18:47.489Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122025-01-15T05:18:47.506Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132025-01-15T05:18:47.506Zdownloading input: /input/build/work/scripts/test_ds.sh
142025-01-15T05:18:47.523Zdownloaded input: /input/build/work/scripts/test_ds.sh
152025-01-15T05:18:47.524Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162025-01-15T05:18:47.548Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172025-01-15T05:18:47.549Zdownloading input: /input/build/work/scripts/test_repair.sh
182025-01-15T05:18:47.570Zdownloaded input: /input/build/work/scripts/test_repair.sh
192025-01-15T05:18:47.570Zdownloading input: /input/build/work/scripts/test_replay.sh
202025-01-15T05:18:47.595Zdownloaded input: /input/build/work/scripts/test_replay.sh
212025-01-15T05:18:47.595Zdownloading input: /input/build/work/scripts/test_up.sh
222025-01-15T05:18:47.615Zdownloaded input: /input/build/work/scripts/test_up.sh
232025-01-15T05:18:47.615Zdownloading input: /input/build/work/scripts/upstairs_info.d
242025-01-15T05:18:47.640Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252025-01-15T05:18:47.646Zdownloading input: /input/build/tmp/cargo-test-out.log
262025-01-15T05:18:48.542Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272025-01-15T05:18:48.542Zstarting task 0: "setup"
282025-01-15T05:18:48.545Z++ uname -s
292025-01-15T05:18:48.545Z+ kern=SunOS
302025-01-15T05:18:48.545Z+ case "$kern" in
312025-01-15T05:18:48.545Z+ groupadd -g 12345 build
322025-01-15T05:18:48.548Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332025-01-15T05:18:50.554Z+ zfs create -o mountpoint=/work rpool/work
342025-01-15T05:18:50.685Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352025-01-15T05:18:50.689Z+ home_fs=zfs
362025-01-15T05:18:50.689Z+ [[ zfs == autofs ]]
372025-01-15T05:18:50.689Z+ mkdir -p /home/build
382025-01-15T05:18:50.692Z+ chown build:build /home/build /work
392025-01-15T05:18:52.692Z+ chmod 0700 /home/build /work
402025-01-15T05:18:52.695Zprocess exited: duration 4164 ms, exit code 0
 
412025-01-15T05:18:52.700Zstarting task 1: "authentication"
422025-01-15T05:18:52.717Zprocess exited: duration 16 ms, exit code 0
 
432025-01-15T05:18:52.723Zstarting task 2: "build"
442025-01-15T05:18:52.728Z+ banner cores
452025-01-15T05:18:52.731Z
462025-01-15T05:18:52.731Z #### #### ##### ###### ####
472025-01-15T05:18:52.734Z # # # # # # # #
482025-01-15T05:18:52.734Z # # # # # ##### ####
492025-01-15T05:18:52.734Z # # # ##### # #
502025-01-15T05:18:52.734Z # # # # # # # # #
512025-01-15T05:18:52.734Z #### #### # # ###### ####
522025-01-15T05:18:52.734Z
532025-01-15T05:18:52.734Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542025-01-15T05:18:52.740Z+ banner unpack
552025-01-15T05:18:52.743Z
562025-01-15T05:18:52.743Z # # # # ##### ## #### # #
572025-01-15T05:18:52.743Z # # ## # # # # # # # # #
582025-01-15T05:18:52.743Z # # # # # # # # # # ####
592025-01-15T05:18:52.743Z # # # # # ##### ###### # # #
602025-01-15T05:18:52.743Z # # # ## # # # # # # #
612025-01-15T05:18:52.743Z #### # # # # # #### # #
622025-01-15T05:18:52.743Z
632025-01-15T05:18:52.743Z+ mkdir -p /var/tmp/bins
642025-01-15T05:18:52.748Z+ for t in "$input/bins/"*.gz
652025-01-15T05:18:52.748Z++ basename /input/build/work/bins/crucible-downstairs.gz
662025-01-15T05:18:52.750Z+ b=crucible-downstairs.gz
672025-01-15T05:18:52.750Z+ b=crucible-downstairs
682025-01-15T05:18:52.750Z+ gunzip
692025-01-15T05:18:55.003Z+ chmod +x /var/tmp/bins/crucible-downstairs
702025-01-15T05:18:55.007Z+ for t in "$input/bins/"*.gz
712025-01-15T05:18:55.010Z++ basename /input/build/work/bins/crucible-hammer.gz
722025-01-15T05:18:55.010Z+ b=crucible-hammer.gz
732025-01-15T05:18:55.010Z+ b=crucible-hammer
742025-01-15T05:18:55.010Z+ gunzip
752025-01-15T05:18:57.146Z+ chmod +x /var/tmp/bins/crucible-hammer
762025-01-15T05:18:57.148Z+ for t in "$input/bins/"*.gz
772025-01-15T05:18:57.151Z++ basename /input/build/work/bins/crutest.gz
782025-01-15T05:18:57.151Z+ b=crutest.gz
792025-01-15T05:18:57.151Z+ b=crutest
802025-01-15T05:18:57.151Z+ gunzip
812025-01-15T05:18:59.517Z+ chmod +x /var/tmp/bins/crutest
822025-01-15T05:18:59.520Z+ for t in "$input/bins/"*.gz
832025-01-15T05:18:59.522Z++ basename /input/build/work/bins/dsc.gz
842025-01-15T05:18:59.523Z+ b=dsc.gz
852025-01-15T05:18:59.523Z+ b=dsc
862025-01-15T05:18:59.523Z+ gunzip
872025-01-15T05:19:00.880Z+ chmod +x /var/tmp/bins/dsc
882025-01-15T05:19:00.883Z+ export BINDIR=/var/tmp/bins
892025-01-15T05:19:00.883Z+ BINDIR=/var/tmp/bins
902025-01-15T05:19:00.883Z+ jobpid=1141
912025-01-15T05:19:00.885Z+ echo 'Setup debug logging'
922025-01-15T05:19:00.885ZSetup debug logging
932025-01-15T05:19:00.885Z+ mkdir /tmp/debug
942025-01-15T05:19:00.886Z+ sleep 3600
952025-01-15T05:19:00.886Z+ psrinfo -v
962025-01-15T05:19:00.889Z+ df -h
972025-01-15T05:19:00.896Z+ prstat -d d -mLc 1
982025-01-15T05:19:00.900Z+ iostat -T d -xn 1
992025-01-15T05:19:00.900Z+ mpstat -T d 1
1002025-01-15T05:19:00.900Z+ vmstat -T d -p 1
1012025-01-15T05:19:00.903Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1022025-01-15T05:19:00.903Z+ banner test_up_2r_encrypted
1032025-01-15T05:19:00.903Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1042025-01-15T05:19:00.903Z #####
1052025-01-15T05:19:00.903Z ##### ###### #### ##### # # ##### # # #####
1062025-01-15T05:19:00.903Z # # # # # # # # # # #
1072025-01-15T05:19:00.903Z # ##### #### # # # # # ##### # #
1082025-01-15T05:19:00.903Z # # # # # # ##### # #####
1092025-01-15T05:19:00.903Z # # # # # # # # # # #
1102025-01-15T05:19:00.903Z # ###### #### # ####### #### # ####### ####### # #
1112025-01-15T05:19:00.903Z
1122025-01-15T05:19:00.903Z+ ptime -m bash /input/build/work/scripts/test_up.sh -r 2 -N encrypted
1132025-01-15T05:19:00.915Z/input/build/work
1142025-01-15T05:19:00.929ZUsing 2 region sets
1152025-01-15T05:19:00.929ZTurn off color for downstairs dump
1162025-01-15T05:19:00.942ZUpstairs using key: 2B/aoxneuwLnOO3UDIfwWS7GN/Od9h66OG3XkVDS2HE=
1172025-01-15T05:19:00.945Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1182025-01-15T05:19:00.945ZCreating 6 downstairs regions
1192025-01-15T05:19:01.329ZStarting 6 downstairs
1202025-01-15T05:19:01.333Zdsc started at PID: 1189
1212025-01-15T05:19:06.698ZDisable automatic restart on all downstairs
1222025-01-15T05:19:06.757Z
1232025-01-15T05:19:06.757ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1242025-01-15T05:19:06.761ZRunning test: span
1252025-01-15T05:19:10.941ZCompleted test: span
1262025-01-15T05:19:15.948ZRunning test: big
1272025-01-15T05:19:20.524ZCompleted test: big
1282025-01-15T05:19:25.531ZRunning test: dep
1292025-01-15T05:19:58.882ZCompleted test: dep
1302025-01-15T05:20:03.890ZRunning test: balloon
1312025-01-15T05:24:16.083ZCompleted test: balloon
1322025-01-15T05:24:21.091ZRunning test: deactivate
1332025-01-15T05:25:15.796ZCompleted test: deactivate
1342025-01-15T05:25:20.804ZRunning hammer
1352025-01-15T05:25:38.857ZRun repair tests
1362025-01-15T05:25:38.861Z/var/tmp/bins/crutest fill -g 31 -q --key 2B/aoxneuwLnOO3UDIfwWS7GN/Od9h66OG3XkVDS2HE= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1372025-01-15T05:25:38.923Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.518279937Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"task":"crutest"}
1382025-01-15T05:25:38.926Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.518899751Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"task":"crutest"}
1392025-01-15T05:25:38.926Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.520671768Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"task":"crutest"}
1402025-01-15T05:25:38.926Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.520896164Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232}
1412025-01-15T05:25:38.926Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.520953441Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232}
1422025-01-15T05:25:38.926Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.520978019Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232}
1432025-01-15T05:25:38.926Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.520993319Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232}
1442025-01-15T05:25:38.927Z{"msg":"Crucible 0a54311e-51e7-4d53-a831-abf53f7f85bf has session id: 866812f8-5394-49eb-9423-2ad8a0b950d3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.52164998Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1452025-01-15T05:25:38.927Z{"msg":"Upstairs opts: Upstairs UUID: 0a54311e-51e7-4d53-a831-abf53f7f85bf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.521706007Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1462025-01-15T05:25:38.927Z{"msg":"Crucible stats registered with UUID: 0a54311e-51e7-4d53-a831-abf53f7f85bf","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.521731326Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1472025-01-15T05:25:38.927Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:37.521761264Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1482025-01-15T05:25:38.931Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.523878701Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"task":"crutest"}
1492025-01-15T05:25:38.931Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.523954856Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232}
1502025-01-15T05:25:38.931Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.523998534Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232}
1512025-01-15T05:25:38.931Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.524014593Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232}
1522025-01-15T05:25:38.931Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.524040471Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232}
1532025-01-15T05:25:38.932Z{"msg":"Crucible 0a54311e-51e7-4d53-a831-abf53f7f85bf has session id: f3b8cf42-d01e-474f-883c-43903a5d2464","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.524493775Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
1542025-01-15T05:25:38.932Z{"msg":"Upstairs opts: Upstairs UUID: 0a54311e-51e7-4d53-a831-abf53f7f85bf, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.524556261Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
1552025-01-15T05:25:38.932Z{"msg":"Crucible stats registered with UUID: 0a54311e-51e7-4d53-a831-abf53f7f85bf","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.524598659Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
1562025-01-15T05:25:38.932Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:37.524626707Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
1572025-01-15T05:25:38.932Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.525484697Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"task":"crutest"}
1582025-01-15T05:25:38.932Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.525530114Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232}
1592025-01-15T05:25:38.932Z{"msg":"0a54311e-51e7-4d53-a831-abf53f7f85bf active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.525545153Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1602025-01-15T05:25:38.932Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.525558493Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"client":"0","":"downstairs","session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1612025-01-15T05:25:38.932Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.525581122Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"client":"0","":"downstairs","session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1622025-01-15T05:25:38.932Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.5256039Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"client":"1","":"downstairs","session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1632025-01-15T05:25:38.933Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.525629399Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"client":"1","":"downstairs","session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1642025-01-15T05:25:38.933Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.525663677Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"client":"2","":"downstairs","session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1652025-01-15T05:25:38.933Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.525694875Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"client":"2","":"downstairs","session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1662025-01-15T05:25:38.933Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.525732313Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1672025-01-15T05:25:38.933Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.52577847Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1682025-01-15T05:25:38.933Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.525805688Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1692025-01-15T05:25:38.933Z{"msg":"ds_connection connected from Ok(127.0.0.1:35729)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.525821648Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1702025-01-15T05:25:38.933Z{"msg":"ds_connection connected from Ok(127.0.0.1:62655)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.525968589Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1712025-01-15T05:25:38.933Z{"msg":"ds_connection connected from Ok(127.0.0.1:44933)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.526111111Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1722025-01-15T05:25:38.933Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.526637Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"client":"2","":"downstairs","session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1732025-01-15T05:25:38.934Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.526675018Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"client":"0","":"downstairs","session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1742025-01-15T05:25:38.934Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.526694477Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"client":"1","":"downstairs","session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1752025-01-15T05:25:38.934Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.527307731Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"client":"2","":"downstairs","session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1762025-01-15T05:25:38.934Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.527345289Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"client":"1","":"downstairs","session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1772025-01-15T05:25:38.934Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.527416344Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"client":"0","":"downstairs","session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1782025-01-15T05:25:38.934Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.527751735Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1792025-01-15T05:25:38.934Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.527789303Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1802025-01-15T05:25:38.934Z{"msg":"[0]R flush_numbers: [4004, 4004, 4004, 4004, 4004]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.527812592Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"downstairs","":"downstairs","session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1812025-01-15T05:25:38.934Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.52784223Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"downstairs","":"downstairs","session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1822025-01-15T05:25:38.934Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.527866368Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"downstairs","":"downstairs","session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1832025-01-15T05:25:38.938Z{"msg":"[1]R flush_numbers: [4004, 4004, 4004, 4004, 4004]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.527891077Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"downstairs","":"downstairs","session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1842025-01-15T05:25:38.938Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.527930985Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"downstairs","":"downstairs","session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1852025-01-15T05:25:38.938Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.527954893Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"downstairs","":"downstairs","session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1862025-01-15T05:25:38.938Z{"msg":"[2]R flush_numbers: [4004, 4004, 4004, 4004, 4004]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.527988531Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"downstairs","":"downstairs","session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1872025-01-15T05:25:38.938Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.52801422Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"downstairs","":"downstairs","session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1882025-01-15T05:25:38.938Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.528047028Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"downstairs","":"downstairs","session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1892025-01-15T05:25:38.938Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.528072186Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"downstairs","":"downstairs","session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1902025-01-15T05:25:38.939Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.528099355Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"downstairs","":"downstairs","session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1912025-01-15T05:25:38.939Z{"msg":"Next flush: 4005","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.528118494Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"downstairs","":"downstairs","session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1922025-01-15T05:25:38.939Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.528137062Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"downstairs","":"downstairs","session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1932025-01-15T05:25:38.939Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.528167581Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1942025-01-15T05:25:38.939Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.528194619Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"client":"0","":"downstairs","session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1952025-01-15T05:25:38.939Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.528228587Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"client":"1","":"downstairs","session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1962025-01-15T05:25:38.939Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.528261715Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"client":"2","":"downstairs","session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1972025-01-15T05:25:38.939Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.528286514Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1982025-01-15T05:25:38.939Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.528316162Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
1992025-01-15T05:25:38.939Z{"msg":"0a54311e-51e7-4d53-a831-abf53f7f85bf is now active with session: 866812f8-5394-49eb-9423-2ad8a0b950d3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.528339341Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
2002025-01-15T05:25:38.939Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.528375849Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"session_id":"866812f8-5394-49eb-9423-2ad8a0b950d3"}
2012025-01-15T05:25:38.939Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.528515771Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232}
2022025-01-15T05:25:38.939Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.528548589Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232}
2032025-01-15T05:25:38.939Z{"msg":"0a54311e-51e7-4d53-a831-abf53f7f85bf active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.528568257Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2042025-01-15T05:25:38.939Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.528600255Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"client":"0","":"downstairs","session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2052025-01-15T05:25:38.940Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.528628504Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"client":"0","":"downstairs","session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2062025-01-15T05:25:38.940Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.528666612Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"client":"1","":"downstairs","session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2072025-01-15T05:25:38.940Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.52869562Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"client":"1","":"downstairs","session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2082025-01-15T05:25:38.940Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.528719539Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"client":"2","":"downstairs","session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2092025-01-15T05:25:38.940Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.528760016Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"client":"2","":"downstairs","session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2102025-01-15T05:25:38.940Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.528789995Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2112025-01-15T05:25:38.940Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.528819883Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2122025-01-15T05:25:38.940Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.528858011Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2132025-01-15T05:25:38.940Z{"msg":"ds_connection connected from Ok(127.0.0.1:42989)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.52887381Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2142025-01-15T05:25:38.940Z{"msg":"ds_connection connected from Ok(127.0.0.1:36370)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.528895218Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2152025-01-15T05:25:38.940Z{"msg":"ds_connection connected from Ok(127.0.0.1:51837)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.528909677Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2162025-01-15T05:25:38.940Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.528940676Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"client":"1","":"downstairs","session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2172025-01-15T05:25:38.940Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.529080898Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"client":"0","":"downstairs","session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2182025-01-15T05:25:38.940Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.529128095Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"client":"2","":"downstairs","session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2192025-01-15T05:25:38.940Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.529316854Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"client":"1","":"downstairs","session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2202025-01-15T05:25:38.940Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.529650444Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2212025-01-15T05:25:38.940Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.529706651Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"client":"2","":"downstairs","session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2222025-01-15T05:25:38.940Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.529787866Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"client":"0","":"downstairs","session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2232025-01-15T05:25:38.940Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.530055131Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2242025-01-15T05:25:38.941Z{"msg":"[0]R flush_numbers: [5323, 5323, 5319, 5319, 5323]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.530090569Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"downstairs","":"downstairs","session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2252025-01-15T05:25:38.941Z{"msg":"[0]R generation: [14, 14, 13, 13, 14]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.530107038Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"downstairs","":"downstairs","session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2262025-01-15T05:25:38.941Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.530120447Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"downstairs","":"downstairs","session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2272025-01-15T05:25:38.941Z{"msg":"[1]R flush_numbers: [5323, 5323, 5319, 5319, 5323]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.530134116Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"downstairs","":"downstairs","session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2282025-01-15T05:25:38.941Z{"msg":"[1]R generation: [14, 14, 13, 13, 14]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.530154725Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"downstairs","":"downstairs","session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2292025-01-15T05:25:38.941Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.530168314Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"downstairs","":"downstairs","session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2302025-01-15T05:25:38.941Z{"msg":"[2]R flush_numbers: [5323, 5323, 5319, 5319, 5323]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.530182014Z","hostname"Wait for a query_work_queue command to finish before sending IO
2312025-01-15T05:25:38.941Z:"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"downstairs","":"downstairs","session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2322025-01-15T05:25:38.941Z{"msg":"[2]R generation: [14, 14, 13, 13, 14]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.530210182Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"downstairs","":"downstairs","session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2332025-01-15T05:25:38.941Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.530223571Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"downstairs","":"downstairs","session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2342025-01-15T05:25:38.941Z{"msg":"Max found gen is 15","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.53023755Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"downstairs","":"downstairs","session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2352025-01-15T05:25:38.942Z{"msg":"Generation requested: 31 >= found:15","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.53025053Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"downstairs","":"downstairs","session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2362025-01-15T05:25:38.942Z{"msg":"Next flush: 5324","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.530271338Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"downstairs","":"downstairs","session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2372025-01-15T05:25:38.942Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.530284677Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"":"downstairs","":"downstairs","session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2382025-01-15T05:25:38.942Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.530298127Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2392025-01-15T05:25:38.942Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.530309736Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"client":"0","":"downstairs","session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2402025-01-15T05:25:38.942Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.530330625Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"client":"1","":"downstairs","session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2412025-01-15T05:25:38.942Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.530343734Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"client":"2","":"downstairs","session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2422025-01-15T05:25:38.942Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.530357113Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2432025-01-15T05:25:38.942Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.530376472Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2442025-01-15T05:25:38.942Z{"msg":"0a54311e-51e7-4d53-a831-abf53f7f85bf is now active with session: f3b8cf42-d01e-474f-883c-43903a5d2464","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.530389831Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2452025-01-15T05:25:38.942Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.53040279Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232,"session_id":"f3b8cf42-d01e-474f-883c-43903a5d2464"}
2462025-01-15T05:25:38.942Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:37.530422809Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232}
2472025-01-15T05:25:38.942ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
2482025-01-15T05:25:38.942ZFill test
2492025-01-15T05:25:38.975ZRead and Verify all blocks (0..100 range:false)
2502025-01-15T05:25:39.055ZWrote out file "/var/tmp/test_up-build/verify_file"
2512025-01-15T05:25:39.055ZCLIENT: Tests done. All submitted work has been ACK'd
2522025-01-15T05:25:39.055Z----------------------------------------------------------------
2532025-01-15T05:25:39.055Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2542025-01-15T05:25:39.055ZStates: Active Active Active
2552025-01-15T05:25:39.055ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2562025-01-15T05:25:39.055Z 1002 Acked Read 50 Done Done Done false
2572025-01-15T05:25:39.055Z STATES DS:0 DS:1 DS:2 TOTAL
2582025-01-15T05:25:39.055Z Sent 0 0 0 0
2592025-01-15T05:25:39.055Z Done 1 1 1 3
2602025-01-15T05:25:39.058Z Skipped 0 0 0 0
2612025-01-15T05:25:39.059Z Error 0 0 0 0
2622025-01-15T05:25:39.059ZLast Flush: 1001 1001 1001
2632025-01-15T05:25:39.059ZDownstairs last five completed: 1001 1000
2642025-01-15T05:25:39.059ZUpstairs last five completed: 1002 1001 1000
2652025-01-15T05:25:39.059Z----------------------------------------------------------------
2662025-01-15T05:25:39.059Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2672025-01-15T05:25:39.059ZStates: Active Active Active
2682025-01-15T05:25:39.059ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2692025-01-15T05:25:39.059Z 1002 Acked Read 50 Done Done Done false
2702025-01-15T05:25:39.059Z STATES DS:0 DS:1 DS:2 TOTAL
2712025-01-15T05:25:39.059Z Sent 0 0 0 0
2722025-01-15T05:25:39.059Z Done 1 1 1 3
2732025-01-15T05:25:39.059Z Skipped 0 0 0 0
2742025-01-15T05:25:39.059Z Error 0 0 0 0
2752025-01-15T05:25:39.059ZLast Flush: 1001 1001 1001
2762025-01-15T05:25:39.059ZDownstairs last five completed: 1001 1000
2772025-01-15T05:25:39.059ZUpstairs last five completed: 1002 1001 1000
2782025-01-15T05:25:39.059ZCLIENT: Up:0 ds:2 act:6
2792025-01-15T05:25:43.056Z----------------------------------------------------------------
2802025-01-15T05:25:43.056Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2812025-01-15T05:25:43.056ZDownstairs last five completed: 1003 1002 1001 1000
2822025-01-15T05:25:43.056ZUpstairs last five completed: 1003 1002 1001 1000
2832025-01-15T05:25:43.059Z----------------------------------------------------------------
2842025-01-15T05:25:43.060Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2852025-01-15T05:25:43.060ZDownstairs last five completed: 1003 1002 1001 1000
2862025-01-15T05:25:43.060ZUpstairs last five completed: 1003 1002 1001 1000
2872025-01-15T05:25:43.060ZCLIENT: Up:0 ds:0 act:6
2882025-01-15T05:25:43.060ZCLIENT: All crucible jobs finished, exiting program
2892025-01-15T05:25:43.060Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:41.653637428Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1232}
2902025-01-15T05:25:43.069ZRepair setup passed
2912025-01-15T05:25:43.073ZCopy the region for /var/tmp/test_up-build/8830
2922025-01-15T05:25:43.076Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2932025-01-15T05:25:43.079Z/var/tmp/bins/crutest fill -g 32 -q --key 2B/aoxneuwLnOO3UDIfwWS7GN/Od9h66OG3XkVDS2HE= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2942025-01-15T05:25:43.136Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.732838304Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"task":"crutest"}
2952025-01-15T05:25:43.139Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.733465567Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"task":"crutest"}
2962025-01-15T05:25:43.139Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.735179888Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"task":"crutest"}
2972025-01-15T05:25:43.139Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.735406314Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238}
2982025-01-15T05:25:43.139Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.735460821Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238}
2992025-01-15T05:25:43.139Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.73548509Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238}
3002025-01-15T05:25:43.139Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.735500869Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238}
3012025-01-15T05:25:43.139Z{"msg":"Crucible 20f97dbb-158a-44de-bd17-1bee256166fb has session id: d0854bc0-850b-4151-ae49-610160e8ba8f","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.736131402Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3022025-01-15T05:25:43.139Z{"msg":"Upstairs opts: Upstairs UUID: 20f97dbb-158a-44de-bd17-1bee256166fb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.736183719Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3032025-01-15T05:25:43.139Z{"msg":"Crucible stats registered with UUID: 20f97dbb-158a-44de-bd17-1bee256166fb","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.736201268Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3042025-01-15T05:25:43.140Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:41.736214727Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3052025-01-15T05:25:43.143Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.738363992Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"task":"crutest"}
3062025-01-15T05:25:43.143Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.738469046Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238}
3072025-01-15T05:25:43.143Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.738510834Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238}
3082025-01-15T05:25:43.143Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.738536652Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238}
3092025-01-15T05:25:43.143Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.738555211Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238}
3102025-01-15T05:25:43.145Z{"msg":"Crucible 20f97dbb-158a-44de-bd17-1bee256166fb has session id: af83e4d2-d00f-4127-bfe1-9b35633c8e3d","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.738924509Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3112025-01-15T05:25:43.145Z{"msg":"Upstairs opts: Upstairs UUID: 20f97dbb-158a-44de-bd17-1bee256166fb, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.738960358Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3122025-01-15T05:25:43.145Z{"msg":"Crucible stats registered with UUID: 20f97dbb-158a-44de-bd17-1bee256166fb","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.739011795Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3132025-01-15T05:25:43.145Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:41.73908664Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3142025-01-15T05:25:43.145Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.739877234Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"task":"crutest"}
3152025-01-15T05:25:43.145Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.739925911Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238}
3162025-01-15T05:25:43.145Z{"msg":"20f97dbb-158a-44de-bd17-1bee256166fb active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.739967279Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3172025-01-15T05:25:43.145Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.739993887Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"client":"0","":"downstairs","session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3182025-01-15T05:25:43.145Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.740018406Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"client":"0","":"downstairs","session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3192025-01-15T05:25:43.145Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.740041255Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"client":"1","":"downstairs","session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3202025-01-15T05:25:43.146Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.740056004Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"client":"1","":"downstairs","session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3212025-01-15T05:25:43.146Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.740077722Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"client":"2","":"downstairs","session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3222025-01-15T05:25:43.146Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.74011643Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"client":"2","":"downstairs","session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3232025-01-15T05:25:43.146Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.740142438Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3242025-01-15T05:25:43.146Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.740169587Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3252025-01-15T05:25:43.150Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.740248143Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3262025-01-15T05:25:43.150Z{"msg":"ds_connection connected from Ok(127.0.0.1:35256)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.740513287Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3272025-01-15T05:25:43.150Z{"msg":"ds_connection connected from Ok(127.0.0.1:45943)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.740556425Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3282025-01-15T05:25:43.150Z{"msg":"ds_connection connected from Ok(127.0.0.1:62727)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.740582263Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3292025-01-15T05:25:43.150Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.741097143Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"client":"1","":"downstairs","session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3302025-01-15T05:25:43.151Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.741136261Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"client":"0","":"downstairs","session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3312025-01-15T05:25:43.151Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.741208656Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"client":"2","":"downstairs","session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3322025-01-15T05:25:43.151Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.741721906Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"client":"1","":"downstairs","session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3332025-01-15T05:25:43.151Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.741768734Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"client":"2","":"downstairs","session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3342025-01-15T05:25:43.151Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.741799072Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"client":"0","":"downstairs","session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3352025-01-15T05:25:43.151Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.742134742Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3362025-01-15T05:25:43.151Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.74218433Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3372025-01-15T05:25:43.151Z{"msg":"[0]R flush_numbers: [4005, 4005, 4005, 4005, 4005]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.742209798Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"downstairs","":"downstairs","session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3382025-01-15T05:25:43.151Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.742246276Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"downstairs","":"downstairs","session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3392025-01-15T05:25:43.151Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.742281104Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"downstairs","":"downstairs","session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3402025-01-15T05:25:43.152Z{"msg":"[1]R flush_numbers: [4005, 4005, 4005, 4005, 4005]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.742304883Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"downstairs","":"downstairs","session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3412025-01-15T05:25:43.152Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.742337521Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"downstairs","":"downstairs","session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3422025-01-15T05:25:43.152Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.742372689Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"downstairs","":"downstairs","session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3432025-01-15T05:25:43.152Z{"msg":"[2]R flush_numbers: [4005, 4005, 4005, 4005, 4005]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.742397757Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"downstairs","":"downstairs","session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3442025-01-15T05:25:43.152Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.742433515Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"downstairs","":"downstairs","session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3452025-01-15T05:25:43.153Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.742475233Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"downstairs","":"downstairs","session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3462025-01-15T05:25:43.153Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.742503301Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"downstairs","":"downstairs","session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3472025-01-15T05:25:43.153Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.742543659Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"downstairs","":"downstairs","session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3482025-01-15T05:25:43.153Z{"msg":"Next flush: 4006","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.742569767Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"downstairs","":"downstairs","session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3492025-01-15T05:25:43.153Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.742599506Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"downstairs","":"downstairs","session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3502025-01-15T05:25:43.153Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.742625304Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3512025-01-15T05:25:43.153Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.742649843Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"client":"0","":"downstairs","session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3522025-01-15T05:25:43.153Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.742676511Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"client":"1","":"downstairs","session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3532025-01-15T05:25:43.153Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.742879719Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"client":"2","":"downstairs","session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3542025-01-15T05:25:43.153Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.742918827Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3552025-01-15T05:25:43.153Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.742937296Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3562025-01-15T05:25:43.153Z{"msg":"20f97dbb-158a-44de-bd17-1bee256166fb is now active with session: d0854bc0-850b-4151-ae49-610160e8ba8f","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.742965704Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3572025-01-15T05:25:43.153Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.742979483Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"session_id":"d0854bc0-850b-4151-ae49-610160e8ba8f"}
3582025-01-15T05:25:43.153Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.74303689Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238}
3592025-01-15T05:25:43.153Z{"msg":"20f97dbb-158a-44de-bd17-1bee256166fb active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.743079047Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3602025-01-15T05:25:43.153Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.743106146Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"client":"0","":"downstairs","session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3612025-01-15T05:25:43.154Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.743135264Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"client":"0","":"downstairs","session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3622025-01-15T05:25:43.154Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.743159763Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"client":"1","":"downstairs","session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3632025-01-15T05:25:43.154Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.743182861Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"client":"1","":"downstairs","session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3642025-01-15T05:25:43.155Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.74320052Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"client":"2","":"downstairs","session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3652025-01-15T05:25:43.155Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.743218789Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"client":"2","":"downstairs","session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3662025-01-15T05:25:43.155Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.743232878Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238}
3672025-01-15T05:25:43.155Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.743244478Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3682025-01-15T05:25:43.155Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.743264517Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3692025-01-15T05:25:43.155Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.743278276Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3702025-01-15T05:25:43.155Z{"msg":"ds_connection connected from Ok(127.0.0.1:43500)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.743299985Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3712025-01-15T05:25:43.156Z{"msg":"ds_connection connected from Ok(127.0.0.1:50402)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.743339152Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3722025-01-15T05:25:43.156Z{"msg":"ds_connection connected from Ok(127.0.0.1:46032)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.74338374Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3732025-01-15T05:25:43.156Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.743423218Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"client":"2","":"downstairs","session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3742025-01-15T05:25:43.156Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.743438997Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"client":"1","":"downstairs","session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3752025-01-15T05:25:43.156Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.743464265Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"client":"0","":"downstairs","session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3762025-01-15T05:25:43.156Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.743619706Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"client":"2","":"downstairs","session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3772025-01-15T05:25:43.156Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.743663224Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"client":"1","":"downstairs","session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3782025-01-15T05:25:43.156Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.743801545Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"client":"0","":"downstairs","session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3792025-01-15T05:25:43.156Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.743976935Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3802025-01-15T05:25:43.156Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.74407364Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3812025-01-15T05:25:43.157Z{"msg":"[0]R flush_numbers: [5324, 5324, 5324, 5324, 5324]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.744166664Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"downstairs","":"downstairs","session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3822025-01-15T05:25:43.157Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.744195353Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"downstairs","":"downstairs","session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3832025-01-15T05:25:43.157Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.74424267Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"downstairs","":"downstairs","session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3842025-01-15T05:25:43.157Z{"msg":"[1]R flush_numbers: [5324, 5324, 5324, 5324, 5324]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.744279877Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"downstairs","":"downstairs","session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3852025-01-15T05:25:43.157Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.744314366Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"downstairs","":"downstairs","session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3862025-01-15T05:25:43.157Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.744338844Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"downstairs"Wait for a query_work_queue command to finish before sending IO
3872025-01-15T05:25:43.157Z,"":"downstairs","session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3882025-01-15T05:25:43.157Z{"msg":"[2]R flush_numbers: [5324, 5324, 5324, 5324, 5324]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.744377202Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"downstairs","":"downstairs","session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3892025-01-15T05:25:43.157Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.744401081Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"downstairs","":"downstairs","session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3902025-01-15T05:25:43.157Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.744431139Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"downstairs","":"downstairs","session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3912025-01-15T05:25:43.157Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.744469166Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"downstairs","":"downstairs","session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3922025-01-15T05:25:43.158Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.744491375Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"downstairs","":"downstairs","session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3932025-01-15T05:25:43.158Z{"msg":"Next flush: 5325","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.744523453Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"downstairs","":"downstairs","session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3942025-01-15T05:25:43.158Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.744543512Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"":"downstairs","":"downstairs","session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3952025-01-15T05:25:43.158Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.744556812Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3962025-01-15T05:25:43.158Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.74458261Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"client":"0","":"downstairs","session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3972025-01-15T05:25:43.158Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.744608198Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"client":"1","":"downstairs","session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3982025-01-15T05:25:43.158Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.744637267Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"client":"2","":"downstairs","session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
3992025-01-15T05:25:43.158Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.744651826Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
4002025-01-15T05:25:43.158Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.744684454Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
4012025-01-15T05:25:43.158Z{"msg":"20f97dbb-158a-44de-bd17-1bee256166fb is now active with session: af83e4d2-d00f-4127-bfe1-9b35633c8e3d","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.744719152Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
4022025-01-15T05:25:43.158Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.744742641Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238,"session_id":"af83e4d2-d00f-4127-bfe1-9b35633c8e3d"}
4032025-01-15T05:25:43.159Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.74475963Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238}
4042025-01-15T05:25:43.159ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
4052025-01-15T05:25:43.159ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4062025-01-15T05:25:43.159ZFill test
4072025-01-15T05:25:43.189ZRead and Verify all blocks (0..100 range:false)
4082025-01-15T05:25:43.267ZWrote out file "/var/tmp/test_up-build/verify_file"
4092025-01-15T05:25:43.267ZCLIENT: Tests done. All submitted work has been ACK'd
4102025-01-15T05:25:43.267Z----------------------------------------------------------------
4112025-01-15T05:25:43.267Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4122025-01-15T05:25:43.267ZStates: Active Active Active
4132025-01-15T05:25:43.267ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4142025-01-15T05:25:43.270Z 1002 Acked Read 50 Done Done Done false
4152025-01-15T05:25:43.270Z STATES DS:0 DS:1 DS:2 TOTAL
4162025-01-15T05:25:43.270Z Sent 0 0 0 0
4172025-01-15T05:25:43.270Z Done 1 1 1 3
4182025-01-15T05:25:43.270Z Skipped 0 0 0 0
4192025-01-15T05:25:43.270Z Error 0 0 0 0
4202025-01-15T05:25:43.270ZLast Flush: 1001 1001 1001
4212025-01-15T05:25:43.270ZDownstairs last five completed: 1001 1000
4222025-01-15T05:25:43.270ZUpstairs last five completed: 1002 1001 1000
4232025-01-15T05:25:43.270Z----------------------------------------------------------------
4242025-01-15T05:25:43.270Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4252025-01-15T05:25:43.270ZStates: Active Active Active
4262025-01-15T05:25:43.270ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4272025-01-15T05:25:43.270Z 1002 Acked Read 50 Done Done Done false
4282025-01-15T05:25:43.270Z STATES DS:0 DS:1 DS:2 TOTAL
4292025-01-15T05:25:43.270Z Sent 0 0 0 0
4302025-01-15T05:25:43.270Z Done 1 1 1 3
4312025-01-15T05:25:43.270Z Skipped 0 0 0 0
4322025-01-15T05:25:43.271Z Error 0 0 0 0
4332025-01-15T05:25:43.271ZLast Flush: 1001 1001 1001
4342025-01-15T05:25:43.271ZDownstairs last five completed: 1001 1000
4352025-01-15T05:25:43.271ZUpstairs last five completed: 1002 1001 1000
4362025-01-15T05:25:43.271ZCLIENT: Up:0 ds:2 act:6
4372025-01-15T05:25:47.268Z----------------------------------------------------------------
4382025-01-15T05:25:47.268Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4392025-01-15T05:25:47.268ZDownstairs last five completed: 1003 1002 1001 1000
4402025-01-15T05:25:47.268ZUpstairs last five completed: 1003 1002 1001 1000
4412025-01-15T05:25:47.268Z----------------------------------------------------------------
4422025-01-15T05:25:47.268Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4432025-01-15T05:25:47.268ZDownstairs last five completed: 1003 1002 1001 1000
4442025-01-15T05:25:47.268ZUpstairs last five completed: 1003 1002 1001 1000
4452025-01-15T05:25:47.272ZCLIENT: Up:0 ds:0 act:6
4462025-01-15T05:25:47.272ZCLIENT: All crucible jobs finished, exiting program
4472025-01-15T05:25:47.272Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:45.865529138Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1238}
4482025-01-15T05:25:47.276ZRepair part 1 passed
4492025-01-15T05:25:47.279Z
4502025-01-15T05:25:47.279ZKill the current downstairs
4512025-01-15T05:25:47.455ZDownstairs 2 stopped
4522025-01-15T05:25:47.455Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
4532025-01-15T05:25:47.458ZNow put back the original so we have a mismatch
4542025-01-15T05:25:47.458Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
4552025-01-15T05:25:47.461ZRestart downstairs with old directory
4562025-01-15T05:25:47.520Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4572025-01-15T05:25:47.539ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4582025-01-15T05:25:47.541Z 0 000-009 32 32 31 4006 4006 4005 F F F <---
4592025-01-15T05:25:47.542Z 1 010-019 32 32 31 4006 4006 4005 F F F <---
4602025-01-15T05:25:47.542Z 2 020-029 32 32 31 4006 4006 4005 F F F <---
4612025-01-15T05:25:47.542Z 3 030-039 32 32 31 4006 4006 4005 F F F <---
4622025-01-15T05:25:47.542Z 4 040-049 32 32 31 4006 4006 4005 F F F <---
4632025-01-15T05:25:47.542ZMax gen: 32, Max flush: 4006
4642025-01-15T05:25:47.542ZError: Difference in extent metadata found!
4652025-01-15T05:25:47.544Zdump test found error as expected
4662025-01-15T05:25:47.544Z
4672025-01-15T05:25:47.544Z
4682025-01-15T05:25:47.544Z/var/tmp/bins/crutest verify --range -g 33 -q --key 2B/aoxneuwLnOO3UDIfwWS7GN/Od9h66OG3XkVDS2HE= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
4692025-01-15T05:25:47.603Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.200474677Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"task":"crutest"}
4702025-01-15T05:25:47.606Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.201118329Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"task":"crutest"}
4712025-01-15T05:25:47.606Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.202807491Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"task":"crutest"}
4722025-01-15T05:25:47.606Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.203054327Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246}
4732025-01-15T05:25:47.606Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.203102964Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246}
4742025-01-15T05:25:47.607Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.203135162Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246}
4752025-01-15T05:25:47.607Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.20316225Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246}
4762025-01-15T05:25:47.610Z{"msg":"Crucible 6e7ede6a-df89-4c58-b5d4-13139eccc531 has session id: 78c263c5-fe66-4962-99d5-5a9fb3d3c592","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.203745027Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
4772025-01-15T05:25:47.610Z{"msg":"Upstairs opts: Upstairs UUID: 6e7ede6a-df89-4c58-b5d4-13139eccc531, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.203791714Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
4782025-01-15T05:25:47.610Z{"msg":"Crucible stats registered with UUID: 6e7ede6a-df89-4c58-b5d4-13139eccc531","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.203807293Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
4792025-01-15T05:25:47.610Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:46.203824212Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
4802025-01-15T05:25:47.610Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.205795037Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"task":"crutest"}
4812025-01-15T05:25:47.610Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.205897631Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246}
4822025-01-15T05:25:47.610Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.205938189Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246}
4832025-01-15T05:25:47.610Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.205967877Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246}
4842025-01-15T05:25:47.610Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.206017964Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246}
4852025-01-15T05:25:47.610Z{"msg":"Crucible 6e7ede6a-df89-4c58-b5d4-13139eccc531 has session id: d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.206373523Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
4862025-01-15T05:25:47.610Z{"msg":"Upstairs opts: Upstairs UUID: 6e7ede6a-df89-4c58-b5d4-13139eccc531, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.20642891Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
4872025-01-15T05:25:47.611Z{"msg":"Crucible stats registered with UUID: 6e7ede6a-df89-4c58-b5d4-13139eccc531","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.206451909Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
4882025-01-15T05:25:47.611Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:46.206479807Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
4892025-01-15T05:25:47.614Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.207315798Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"task":"crutest"}
4902025-01-15T05:25:47.614Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.207375755Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246}
4912025-01-15T05:25:47.614Z{"msg":"6e7ede6a-df89-4c58-b5d4-13139eccc531 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.207426042Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
4922025-01-15T05:25:47.615Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.20746488Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"0","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
4932025-01-15T05:25:47.615Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.207519127Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"0","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
4942025-01-15T05:25:47.615Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.20762657Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"1","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
4952025-01-15T05:25:47.615Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.207673438Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"1","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
4962025-01-15T05:25:47.615Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.207700966Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"2","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
4972025-01-15T05:25:47.615Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.207727844Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"2","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
4982025-01-15T05:25:47.615Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.207837588Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
4992025-01-15T05:25:47.615Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.207925763Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5002025-01-15T05:25:47.615Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.20797399Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5012025-01-15T05:25:47.615Z{"msg":"ds_connection connected from Ok(127.0.0.1:55026)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.208003988Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5022025-01-15T05:25:47.615Z{"msg":"ds_connection connected from Ok(127.0.0.1:37005)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.208040666Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5032025-01-15T05:25:47.615Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:46.208075714Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5042025-01-15T05:25:47.615Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:46.208094093Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5052025-01-15T05:25:47.615Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:46.208122032Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5062025-01-15T05:25:47.615Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.20814358Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5072025-01-15T05:25:47.615Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.208475751Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"0","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5082025-01-15T05:25:47.615Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.208555816Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"1","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5092025-01-15T05:25:47.615Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.209120023Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"1","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5102025-01-15T05:25:47.615Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.209157091Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"0","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5112025-01-15T05:25:47.616Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.20951854Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5122025-01-15T05:25:47.616Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:46.209562488Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5132025-01-15T05:25:57.613Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.209485465Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5142025-01-15T05:25:57.613Z{"msg":"ds_connection connected from Ok(127.0.0.1:42421)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.209583339Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5152025-01-15T05:25:57.617Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.210721203Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"2","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5162025-01-15T05:25:57.617Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.211273711Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"2","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5172025-01-15T05:25:57.617Z{"msg":"[0]R flush_numbers: [4006, 4006, 4006, 4006, 4006]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.211661158Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5182025-01-15T05:25:57.617Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.211691516Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5192025-01-15T05:25:57.617Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.211707706Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5202025-01-15T05:25:57.617Z{"msg":"[1]R flush_numbers: [4006, 4006, 4006, 4006, 4006]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.211721625Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5212025-01-15T05:25:57.620Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.211743773Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5222025-01-15T05:25:57.620Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.211758413Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5232025-01-15T05:25:57.620Z{"msg":"[2]R flush_numbers: [4005, 4005, 4005, 4005, 4005]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.211772112Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5242025-01-15T05:25:57.620Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.211792731Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5252025-01-15T05:25:57.620Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.21180629Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5262025-01-15T05:25:57.620Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.211819999Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5272025-01-15T05:25:57.620Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.211834128Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5282025-01-15T05:25:57.621Z{"msg":"Next flush: 4007","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.211846878Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5292025-01-15T05:25:57.621Z{"msg":"Extent 0 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.211860727Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5302025-01-15T05:25:57.621Z{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.211875296Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5312025-01-15T05:25:57.622Z{"msg":"extent:0 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.211889325Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5322025-01-15T05:25:57.622Z{"msg":"extent:0 flush: 4006 4006 4005 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.211911434Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5332025-01-15T05:25:57.622Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.211926383Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5342025-01-15T05:25:57.622Z{"msg":"extent:0 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.211951051Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5352025-01-15T05:25:57.622Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.21196929Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5362025-01-15T05:25:57.622Z{"msg":"find dest for source 0 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.211996459Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5372025-01-15T05:25:57.622Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212020867Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5382025-01-15T05:25:57.622Z{"msg":"Extent 1 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212052026Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5392025-01-15T05:25:57.622Z{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212067774Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5402025-01-15T05:25:57.622Z{"msg":"extent:1 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212091513Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5412025-01-15T05:25:57.622Z{"msg":"extent:1 flush: 4006 4006 4005 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212106902Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5422025-01-15T05:25:57.622Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212120842Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5432025-01-15T05:25:57.622Z{"msg":"extent:1 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.21214205Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5442025-01-15T05:25:57.622Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212157169Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5452025-01-15T05:25:57.622Z{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212172399Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5462025-01-15T05:25:57.622Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212193507Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5472025-01-15T05:25:57.622Z{"msg":"Extent 2 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212209656Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5482025-01-15T05:25:57.622Z{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212231705Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5492025-01-15T05:25:57.622Z{"msg":"extent:2 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212246504Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5502025-01-15T05:25:57.622Z{"msg":"extent:2 flush: 4006 4006 4005 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212276022Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5512025-01-15T05:25:57.623Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212302541Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5522025-01-15T05:25:57.623Z{"msg":"extent:2 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212328139Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5532025-01-15T05:25:57.623Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212368677Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5542025-01-15T05:25:57.623Z{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212399215Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5552025-01-15T05:25:57.623Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212427774Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5562025-01-15T05:25:57.623Z{"msg":"Extent 3 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212457182Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5572025-01-15T05:25:57.623Z{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.21248055Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5582025-01-15T05:25:57.623Z{"msg":"extent:3 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.21249635Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5592025-01-15T05:25:57.623Z{"msg":"extent:3 flush: 4006 4006 4005 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212527608Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5602025-01-15T05:25:57.623Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212549766Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5612025-01-15T05:25:57.627Z{"msg":"extent:3 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212565506Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5622025-01-15T05:25:57.627Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212586874Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5632025-01-15T05:25:57.627Z{"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212612133Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5642025-01-15T05:25:57.627Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212646621Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5652025-01-15T05:25:57.627Z{"msg":"Extent 4 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212673689Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5662025-01-15T05:25:57.627Z{"msg":"First source client ID for extent 4","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212704877Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5672025-01-15T05:25:57.627Z{"msg":"extent:4 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212729706Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5682025-01-15T05:25:57.627Z{"msg":"extent:4 flush: 4006 4006 4005 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212758514Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5692025-01-15T05:25:57.627Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212782893Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5702025-01-15T05:25:57.627Z{"msg":"extent:4 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212814911Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5712025-01-15T05:25:57.628Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212840369Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5722025-01-15T05:25:57.628Z{"msg":"find dest for source 0 for extent at index 4","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212867658Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5732025-01-15T05:25:57.628Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212891737Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5742025-01-15T05:25:57.628Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212941194Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"0","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5752025-01-15T05:25:57.628Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.212972322Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"1","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5762025-01-15T05:25:57.628Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.21299638Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"2","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5772025-01-15T05:25:57.628Z{"msg":"Full repair list: {ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.213028069Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5782025-01-15T05:25:57.628Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 4007, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 4007, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 4007, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 4007, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 4007, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.213137012Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5792025-01-15T05:25:57.628Z{"msg":"starting reconciliation 657d993d-6232-4a58-9952-74db24cb6485: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.21317487Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5802025-01-15T05:25:57.629Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.213196539Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"2","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5812025-01-15T05:25:57.629Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.222217493Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"2","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5822025-01-15T05:25:57.632Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.225949166Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"2","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5832025-01-15T05:25:57.634Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.229555446Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"2","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5842025-01-15T05:25:57.637Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.233506226Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"2","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5852025-01-15T05:25:57.640Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.23669245Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"downstairs","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5862025-01-15T05:25:57.640Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.236734738Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"0","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5872025-01-15T05:25:57.640Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.236751537Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"1","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5882025-01-15T05:25:57.640Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.236768896Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"2","":"downstairs","session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5892025-01-15T05:25:57.643Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.236802984Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5902025-01-15T05:25:57.644Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.236826542Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5912025-01-15T05:25:57.644Z{"msg":"6e7ede6a-df89-4c58-b5d4-13139eccc531 is now active with session: 78c263c5-fe66-4962-99d5-5a9fb3d3c592","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.23685837Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"session_id":"78c263c5-fe66-4962-99d5-5a9fb3d3c592"}
5922025-01-15T05:25:57.644Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.236882879Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246}
5932025-01-15T05:25:57.644Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.236916847Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246}
5942025-01-15T05:25:57.644Z{"msg":"6e7ede6a-df89-4c58-b5d4-13139eccc531 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.236931176Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
5952025-01-15T05:25:57.644Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.236951105Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"0","":"downstairs","session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
5962025-01-15T05:25:57.644Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.236966364Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"0","":"downstairs","session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
5972025-01-15T05:25:57.644Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.236980163Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"1","":"downstairs","session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
5982025-01-15T05:25:57.645Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.237003572Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"1","":"downstairs","session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
5992025-01-15T05:25:57.645Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.23703947Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"2","":"downstairs","session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6002025-01-15T05:25:57.645Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.237067748Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"2","":"downstairs","session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6012025-01-15T05:25:57.646Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.237095287Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6022025-01-15T05:25:57.646Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.237123095Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6032025-01-15T05:25:57.646Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.237151713Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6042025-01-15T05:25:57.646Z{"msg":"ds_connection connected from Ok(127.0.0.1:54965)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.237178222Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6052025-01-15T05:25:57.646Z{"msg":"ds_connection connected from Ok(127.0.0.1:59606)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.23720237Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6062025-01-15T05:25:57.646Z{"msg":"ds_connection connected from Ok(127.0.0.1:42378)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.237225779Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6072025-01-15T05:25:57.646Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.237706541Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"1","":"downstairs","session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6082025-01-15T05:25:57.646Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.237762828Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"0","":"downstairs","session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6092025-01-15T05:25:57.646Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.237793746Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"2","":"downstairs","session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6102025-01-15T05:25:57.646Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.238293587Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"1","":"downstairs","session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6112025-01-15T05:25:57.646Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.238335515Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"2","":"downstairs","session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6122025-01-15T05:25:57.646Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.238362243Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"0","":"downstairs","session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6132025-01-15T05:25:57.646Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.238717952Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6142025-01-15T05:25:57.646Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.23875113Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6152025-01-15T05:25:57.646Z{"msg":"[0]R flush_numbers: [5325, 5325, 5325, 5325, 5325]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.238765919Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"downstairs","":"downstairs","session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6162025-01-15T05:25:57.646Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.238786098Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"downstairs","":"downstairs","session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6172025-01-15T05:25:57.646Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.238809767Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"downstairs","":"downstairs","session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6182025-01-15T05:25:57.646Z{"msg":"[1]R flush_numbers: [5325, 5325, 5325, 5325, 5325]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.238839195Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"downstairs","":"downstairs","session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6192025-01-15T05:25:57.646Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO
6202025-01-15T05:25:57.646Ztime":"2025-01-15T05:25:56.238863334Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"downstairs","":"downstairs","session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6212025-01-15T05:25:57.646Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.238903891Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"downstairs","":"downstairs","session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6222025-01-15T05:25:57.646Z{"msg":"[2]R flush_numbers: [5325, 5325, 5325, 5325, 5325]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.238936819Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"downstairs","":"downstairs","session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6232025-01-15T05:25:57.646Z{"msg":"[2]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.238960318Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"downstairs","":"downstairs","session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6242025-01-15T05:25:57.646Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.238992486Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"downstairs","":"downstairs","session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6252025-01-15T05:25:57.646Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.239015785Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"downstairs","":"downstairs","session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6262025-01-15T05:25:57.646Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.239042623Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"downstairs","":"downstairs","session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6272025-01-15T05:25:57.646Z{"msg":"Next flush: 5326","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.239064972Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"downstairs","":"downstairs","session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6282025-01-15T05:25:57.646Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.23909143Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"":"downstairs","":"downstairs","session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6292025-01-15T05:25:57.646Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.239114269Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6302025-01-15T05:25:57.646Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.239140217Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"0","":"downstairs","session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6312025-01-15T05:25:57.647Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.239163836Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"1","":"downstairs","session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6322025-01-15T05:25:57.647Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.239190225Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"client":"2","":"downstairs","session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6332025-01-15T05:25:57.648Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.239213333Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6342025-01-15T05:25:57.648Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.239239722Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6352025-01-15T05:25:57.648Z{"msg":"6e7ede6a-df89-4c58-b5d4-13139eccc531 is now active with session: d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.239262181Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6362025-01-15T05:25:57.648Z{"msg":"Set Active after no reconciliation","v":0Disk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
6372025-01-15T05:25:57.648Z,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.239291959Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246,"session_id":"d0fcb1e5-70c0-4bc8-a264-c1fb6c32ba8e"}
6382025-01-15T05:25:57.648Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:56.239313867Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246}
6392025-01-15T05:25:57.648ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
6402025-01-15T05:25:57.651ZRead and Verify all blocks (0..100 range:true)
6412025-01-15T05:25:57.722ZWrote out file "/var/tmp/test_up-build/verify_file"
6422025-01-15T05:25:57.722ZVerify test completed
6432025-01-15T05:25:57.725ZWrote out file "/var/tmp/test_up-build/verify_file"
6442025-01-15T05:25:57.725ZCLIENT: Tests done. All submitted work has been ACK'd
6452025-01-15T05:25:57.725Z----------------------------------------------------------------
6462025-01-15T05:25:57.725Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6472025-01-15T05:25:57.725ZStates: Active Active Active
6482025-01-15T05:25:57.725ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6492025-01-15T05:25:57.725Z 1000 Acked Read 50 Done Done Done false
6502025-01-15T05:25:57.725Z STATES DS:0 DS:1 DS:2 TOTAL
6512025-01-15T05:25:57.725Z Sent 0 0 0 0
6522025-01-15T05:25:57.725Z Done 1 1 1 3
6532025-01-15T05:25:57.726Z Skipped 0 0 0 0
6542025-01-15T05:25:57.726Z Error 0 0 0 0
6552025-01-15T05:25:57.726ZLast Flush: 0 0 0
6562025-01-15T05:25:57.726ZDownstairs last five completed:
6572025-01-15T05:25:57.726ZUpstairs last five completed: 1000
6582025-01-15T05:25:57.726Z----------------------------------------------------------------
6592025-01-15T05:25:57.726Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6602025-01-15T05:25:57.726ZStates: Active Active Active
6612025-01-15T05:25:57.726ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6622025-01-15T05:25:57.726Z 1000 Acked Read 50 Done Done Done false
6632025-01-15T05:25:57.726Z STATES DS:0 DS:1 DS:2 TOTAL
6642025-01-15T05:25:57.726Z Sent 0 0 0 0
6652025-01-15T05:25:57.726Z Done 1 1 1 3
6662025-01-15T05:25:57.726Z Skipped 0 0 0 0
6672025-01-15T05:25:57.726Z Error 0 0 0 0
6682025-01-15T05:25:57.726ZLast Flush: 0 0 0
6692025-01-15T05:25:57.726ZDownstairs last five completed:
6702025-01-15T05:25:57.726ZUpstairs last five completed: 1000
6712025-01-15T05:25:57.726ZCLIENT: Up:0 ds:2 act:6
6722025-01-15T05:26:01.724Z----------------------------------------------------------------
6732025-01-15T05:26:01.724Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6742025-01-15T05:26:01.724ZDownstairs last five completed: 1001 1000
6752025-01-15T05:26:01.724ZUpstairs last five completed: 1001 1000
6762025-01-15T05:26:01.726Z----------------------------------------------------------------
6772025-01-15T05:26:01.727Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6782025-01-15T05:26:01.727ZDownstairs last five completed: 1001 1000
6792025-01-15T05:26:01.727ZUpstairs last five completed: 1001 1000
6802025-01-15T05:26:01.727ZCLIENT: Up:0 ds:0 act:6
6812025-01-15T05:26:01.727ZCLIENT: All crucible jobs finished, exiting program
6822025-01-15T05:26:01.727Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:26:00.320060021Z","hostname":"w-01JHM6AXXXDCHCBNQWF5TVW69Q","pid":1246}
6832025-01-15T05:26:01.732ZRepair part 2 passed
6842025-01-15T05:26:01.734Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
6852025-01-15T05:26:01.751ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6862025-01-15T05:26:01.751Z 0 000-009 32 32 32 4006 4006 4006 F F F
6872025-01-15T05:26:01.751Z 1 010-019 32 32 32 4006 4006 4006 F F F
6882025-01-15T05:26:01.751Z 2 020-029 32 32 32 4006 4006 4006 F F F
6892025-01-15T05:26:01.751Z 3 030-039 32 32 32 4006 4006 4006 F F F
6902025-01-15T05:26:01.751Z 4 040-049 32 32 32 4006 4006 4006 F F F
6912025-01-15T05:26:01.754ZMax gen: 32, Max flush: 4006
6922025-01-15T05:26:01.757Zdump test passed
6932025-01-15T05:26:01.760Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
6942025-01-15T05:26:01.777Z Extent 1
6952025-01-15T05:26:01.779ZGEN 32 32 32
6962025-01-15T05:26:01.779ZFLUSH_ID 4006 4006 4006
6972025-01-15T05:26:01.779ZDIRTY
6982025-01-15T05:26:01.779Z
6992025-01-15T05:26:01.779ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
7002025-01-15T05:26:01.779Z010 A A A A A A
7012025-01-15T05:26:01.782Z011 A A A A A A
7022025-01-15T05:26:01.783Z012 A A A A A A
7032025-01-15T05:26:01.785Z013 A A A A A A
7042025-01-15T05:26:01.785Z014 A A A A A A
7052025-01-15T05:26:01.786Z015 A A A A A A
7062025-01-15T05:26:01.788Z016 A A A A A A
7072025-01-15T05:26:01.788Z017 A A A A A A
7082025-01-15T05:26:01.790Z018 A A A A A A
7092025-01-15T05:26:01.790Z019 A A A A A A
7102025-01-15T05:26:01.796Zdump extent test passed
7112025-01-15T05:26:01.799Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
7122025-01-15T05:26:01.815ZExtent 2 Block in extent 0 Actual block 20
7132025-01-15T05:26:01.815Z
7142025-01-15T05:26:01.818Z DATA SHA256 VER
7152025-01-15T05:26:01.818Z------ ---------------------------------------------------------------- ---
7162025-01-15T05:26:01.821Z 0 c94e98307023ce08793169bee252b073e450103de430356c2e0830d129afe7ae A
7172025-01-15T05:26:01.821Z 1 c94e98307023ce08793169bee252b073e450103de430356c2e0830d129afe7ae A
7182025-01-15T05:26:01.821Z 2 c94e98307023ce08793169bee252b073e450103de430356c2e0830d129afe7ae A
7192025-01-15T05:26:01.821Z
7202025-01-15T05:26:01.821ZNONCES 0 1 2 DIFF
7212025-01-15T05:26:01.821Z------ ------------------------ ------------------------ ------------------------ -----
7222025-01-15T05:26:01.821Z 0 b213cf30824b9231afb7b241 b213cf30824b9231afb7b241 b213cf30824b9231afb7b241
7232025-01-15T05:26:01.821Z
7242025-01-15T05:26:01.821Z TAGS 0 1 2 DIFF
7252025-01-15T05:26:01.821Z------ -------------------------------- -------------------------------- -------------------------------- -----
7262025-01-15T05:26:01.821Z 0 c39e50d3b6660dd841b6f9f391c0984e c39e50d3b6660dd841b6f9f391c0984e c39e50d3b6660dd841b6f9f391c0984e
7272025-01-15T05:26:01.822Z
7282025-01-15T05:26:01.822ZHASHES 0 1 2 DIFF
7292025-01-15T05:26:01.822Z------ ---------------- ---------------- ---------------- -----
7302025-01-15T05:26:01.822Z
7312025-01-15T05:26:01.825Zdump block test passed
7322025-01-15T05:26:01.825ZInitial upstairs tests have completed, stopping all downstairs
7332025-01-15T05:26:06.895ZCreating 7 larger downstairs regions
7342025-01-15T05:26:07.107ZStarting 7 downstairs
7352025-01-15T05:26:12.114Zdsc restarted at PID: 1262
7362025-01-15T05:26:12.542ZNow do the replace-reconcile test
7372025-01-15T05:26:12.602ZUsing 8870 for the replacement port
7382025-01-15T05:29:05.018ZCompleted test: replace-reconcile
7392025-01-15T05:29:05.018ZNow do the replace-before-active test
7402025-01-15T05:32:38.749ZCompleted test: replace-before-active
7412025-01-15T05:32:38.751ZAll tests have completed, stopping all downstairs
7422025-01-15T05:32:38.812Z
7432025-01-15T05:32:38.812ZAll Tests have passed
7442025-01-15T05:32:38.814Z13:38 Test duration
7452025-01-15T05:32:38.814Z
7462025-01-15T05:32:38.815Zreal 13:37.838310063
7472025-01-15T05:32:38.815Zuser 18:36.113336944
7482025-01-15T05:32:38.815Zsys 27.274708308
7492025-01-15T05:32:38.815Ztrap 0.571798408
7502025-01-15T05:32:38.815Ztflt 0.042090156
7512025-01-15T05:32:38.815Zdflt 0.020529078
7522025-01-15T05:32:38.815Zkflt 0.001047416
7532025-01-15T05:32:38.815Zlock 25:40:20.833809721
7542025-01-15T05:32:38.815Zslp 1:14:50.094241020
7552025-01-15T05:32:38.815Zlat 2:02.157207895
7562025-01-15T05:32:38.815Zstop 0.291013022
7572025-01-15T05:32:38.815Z+ echo 'test-up-2region-encrypted ends'
7582025-01-15T05:32:38.815Ztest-up-2region-encrypted ends
7592025-01-15T05:32:43.818Zprocess exited: duration 826022 ms, exit code 0
7602025-01-15T05:32:43.818Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
7612025-01-15T05:33:43.853Zexec warning: : stdout descriptor may be held open by a background process; giving up!
7622025-01-15T05:33:43.853Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
7632025-01-15T05:33:43.864Zfound 17 output files
7642025-01-15T05:33:43.864Zuploading: /tmp/test_up-build/dsc-out.txt (50515 bytes)
7652025-01-15T05:33:44.875Zuploaded: /tmp/test_up-build/dsc-out.txt
7662025-01-15T05:33:44.877Zuploading: /tmp/test_up-build/test_up_out.txt (9386025 bytes)
7672025-01-15T05:33:46.019Zuploaded: /tmp/test_up-build/test_up_out.txt
7682025-01-15T05:33:46.022Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (147263 bytes)
7692025-01-15T05:33:47.037Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
7702025-01-15T05:33:47.037Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (144635 bytes)
7712025-01-15T05:33:48.054Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
7722025-01-15T05:33:48.057Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (251117 bytes)
7732025-01-15T05:33:49.076Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
7742025-01-15T05:33:49.079Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (139422 bytes)
7752025-01-15T05:33:50.094Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
7762025-01-15T05:33:50.094Zuploading: /tmp/test_up-build/dsc/downstairs-8850.txt (358353 bytes)
7772025-01-15T05:33:51.111Zuploaded: /tmp/test_up-build/dsc/downstairs-8850.txt
7782025-01-15T05:33:51.113Zuploading: /tmp/test_up-build/dsc/downstairs-8860.txt (33786 bytes)
7792025-01-15T05:33:52.121Zuploaded: /tmp/test_up-build/dsc/downstairs-8860.txt
7802025-01-15T05:33:52.124Zuploading: /tmp/test_up-build/dsc/downstairs-8870.txt (730862 bytes)
7812025-01-15T05:33:53.143Zuploaded: /tmp/test_up-build/dsc/downstairs-8870.txt
7822025-01-15T05:33:53.145Zuploading: /tmp/debug/df.txt (1270 bytes)
7832025-01-15T05:33:54.151Zuploaded: /tmp/debug/df.txt
7842025-01-15T05:33:54.154Zuploading: /tmp/debug/dtrace.txt (969188 bytes)
7852025-01-15T05:33:55.175Zuploaded: /tmp/debug/dtrace.txt
7862025-01-15T05:33:55.177Zuploading: /tmp/debug/iostat.txt (264962 bytes)
7872025-01-15T05:33:55.184Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 264962 -> 268562
7882025-01-15T05:33:56.190Zuploaded: /tmp/debug/iostat.txt
7892025-01-15T05:33:56.193Zuploading: /tmp/debug/mpstat.txt (643719 bytes)
7902025-01-15T05:33:56.204Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 643719 -> 653196
7912025-01-15T05:33:57.210Zuploaded: /tmp/debug/mpstat.txt
7922025-01-15T05:33:57.215Zuploading: /tmp/debug/paging.txt (115607 bytes)
7932025-01-15T05:33:57.218Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 115607 -> 117375
7942025-01-15T05:33:58.347Zuploaded: /tmp/debug/paging.txt
7952025-01-15T05:33:58.347Zuploading: /tmp/debug/prstat.txt (1256318 bytes)
7962025-01-15T05:33:58.466Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 1256318 -> 1274407
7972025-01-15T05:33:59.502Zuploaded: /tmp/debug/prstat.txt
7982025-01-15T05:33:59.504Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
7992025-01-15T05:34:00.511Zuploaded: /tmp/debug/psrinfo.txt
8002025-01-15T05:34:00.513Zuploading: /tmp/debug/upstairs-info.txt (193870 bytes)
8012025-01-15T05:34:01.524Zuploaded: /tmp/debug/upstairs-info.txt