01JMKK62MPM84H4ZWB1SB8MCP7: test-up-2region-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01JMKK6EQY02B6Q2YT20CH4EFT

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12025-02-21T06:43:46.111Zjob dependencies complete; ready to run (waiting for 13 m 22 s)
22025-02-21T06:44:48.388Zjob assigned to worker 01JMKKZ20668FRTV1N5DG1TT8J [factory aws, i-0cca40e4a15c5250e] (queued for 1 m 2 s)
32025-02-21T06:44:56.174Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42025-02-21T06:44:57.301Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52025-02-21T06:44:57.301Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62025-02-21T06:44:58.341Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72025-02-21T06:44:58.341Zdownloading input: /input/build/work/bins/crutest.gz
82025-02-21T06:44:59.800Zdownloaded input: /input/build/work/bins/crutest.gz
92025-02-21T06:44:59.800Zdownloading input: /input/build/work/bins/dsc.gz
102025-02-21T06:45:00.455Zdownloaded input: /input/build/work/bins/dsc.gz
112025-02-21T06:45:00.458Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122025-02-21T06:45:00.473Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132025-02-21T06:45:00.473Zdownloading input: /input/build/work/scripts/test_ds.sh
142025-02-21T06:45:00.490Zdownloaded input: /input/build/work/scripts/test_ds.sh
152025-02-21T06:45:00.490Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162025-02-21T06:45:00.508Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172025-02-21T06:45:00.508Zdownloading input: /input/build/work/scripts/test_repair.sh
182025-02-21T06:45:00.533Zdownloaded input: /input/build/work/scripts/test_repair.sh
192025-02-21T06:45:00.533Zdownloading input: /input/build/work/scripts/test_replay.sh
202025-02-21T06:45:00.551Zdownloaded input: /input/build/work/scripts/test_replay.sh
212025-02-21T06:45:00.551Zdownloading input: /input/build/work/scripts/test_up.sh
222025-02-21T06:45:00.570Zdownloaded input: /input/build/work/scripts/test_up.sh
232025-02-21T06:45:00.570Zdownloading input: /input/build/work/scripts/upstairs_info.d
242025-02-21T06:45:00.590Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252025-02-21T06:45:00.593Zdownloading input: /input/build/tmp/cargo-test-out.log
262025-02-21T06:45:01.250Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272025-02-21T06:45:01.250Zstarting task 0: "setup"
282025-02-21T06:45:01.257Z++ uname -s
292025-02-21T06:45:01.260Z+ kern=SunOS
302025-02-21T06:45:01.260Z+ build_user=build
312025-02-21T06:45:01.260Z+ build_uid=12345
322025-02-21T06:45:01.260Z+ work_dir=/work
332025-02-21T06:45:01.260Z+ input_dir=/input
342025-02-21T06:45:01.260Z+ [[ 0 == 12345 ]]
352025-02-21T06:45:01.260Z+ case "$kern" in
362025-02-21T06:45:01.260Z+ groupadd -g 12345 build
372025-02-21T06:45:01.268Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382025-02-21T06:45:03.272Z+ zfs create -o mountpoint=/work rpool/work
392025-02-21T06:45:03.476Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402025-02-21T06:45:03.482Z+ home_fs=zfs
412025-02-21T06:45:03.482Z+ [[ zfs == autofs ]]
422025-02-21T06:45:03.482Z+ mkdir -p /home/build
432025-02-21T06:45:03.482Z+ chown build:build /home/build /work
442025-02-21T06:45:05.483Z+ chmod 0700 /home/build /work
452025-02-21T06:45:05.486Zprocess exited: duration 4235 ms, exit code 0
 
462025-02-21T06:45:05.495Zstarting task 1: "authentication"
472025-02-21T06:45:05.516Zprocess exited: duration 21 ms, exit code 0
 
482025-02-21T06:45:05.523Zstarting task 2: "build"
492025-02-21T06:45:05.529Z+ banner cores
502025-02-21T06:45:05.532Z
512025-02-21T06:45:05.532Z #### #### ##### ###### ####
522025-02-21T06:45:05.532Z # # # # # # # #
532025-02-21T06:45:05.532Z # # # # # ##### ####
542025-02-21T06:45:05.532Z # # # ##### # #
552025-02-21T06:45:05.532Z # # # # # # # # #
562025-02-21T06:45:05.533Z #### #### # # ###### ####
572025-02-21T06:45:05.533Z
582025-02-21T06:45:05.533Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
592025-02-21T06:45:05.541Z+ banner unpack
602025-02-21T06:45:05.546Z
612025-02-21T06:45:05.546Z # # # # ##### ## #### # #
622025-02-21T06:45:05.546Z # # ## # # # # # # # # #
632025-02-21T06:45:05.546Z # # # # # # # # # # ####
642025-02-21T06:45:05.546Z # # # # # ##### ###### # # #
652025-02-21T06:45:05.547Z # # # ## # # # # # # #
662025-02-21T06:45:05.547Z #### # # # # # #### # #
672025-02-21T06:45:05.547Z
682025-02-21T06:45:05.547Z+ mkdir -p /var/tmp/bins
692025-02-21T06:45:05.552Z+ for t in "$input/bins/"*.gz
702025-02-21T06:45:05.552Z++ basename /input/build/work/bins/crucible-downstairs.gz
712025-02-21T06:45:05.552Z+ b=crucible-downstairs.gz
722025-02-21T06:45:05.552Z+ b=crucible-downstairs
732025-02-21T06:45:05.552Z+ gunzip
742025-02-21T06:45:07.840Z+ chmod +x /var/tmp/bins/crucible-downstairs
752025-02-21T06:45:07.843Z+ for t in "$input/bins/"*.gz
762025-02-21T06:45:07.845Z++ basename /input/build/work/bins/crucible-hammer.gz
772025-02-21T06:45:07.845Z+ b=crucible-hammer.gz
782025-02-21T06:45:07.845Z+ b=crucible-hammer
792025-02-21T06:45:07.845Z+ gunzip
802025-02-21T06:45:09.917Z+ chmod +x /var/tmp/bins/crucible-hammer
812025-02-21T06:45:09.920Z+ for t in "$input/bins/"*.gz
822025-02-21T06:45:09.922Z++ basename /input/build/work/bins/crutest.gz
832025-02-21T06:45:09.923Z+ b=crutest.gz
842025-02-21T06:45:09.923Z+ b=crutest
852025-02-21T06:45:09.923Z+ gunzip
862025-02-21T06:45:12.222Z+ chmod +x /var/tmp/bins/crutest
872025-02-21T06:45:12.225Z+ for t in "$input/bins/"*.gz
882025-02-21T06:45:12.228Z++ basename /input/build/work/bins/dsc.gz
892025-02-21T06:45:12.228Z+ b=dsc.gz
902025-02-21T06:45:12.228Z+ b=dsc
912025-02-21T06:45:12.228Z+ gunzip
922025-02-21T06:45:13.554Z+ chmod +x /var/tmp/bins/dsc
932025-02-21T06:45:13.557Z+ export BINDIR=/var/tmp/bins
942025-02-21T06:45:13.557Z+ BINDIR=/var/tmp/bins
952025-02-21T06:45:13.557Z+ jobpid=1150
962025-02-21T06:45:13.560Z+ echo 'Setup debug logging'
972025-02-21T06:45:13.560ZSetup debug logging
982025-02-21T06:45:13.560Z+ mkdir /tmp/debug
992025-02-21T06:45:13.560Z+ sleep 3600
1002025-02-21T06:45:13.560Z+ psrinfo -v
1012025-02-21T06:45:13.563Z+ df -h
1022025-02-21T06:45:13.570Z+ prstat -d d -mLc 1
1032025-02-21T06:45:13.573Z+ iostat -T d -xn 1
1042025-02-21T06:45:13.573Z+ mpstat -T d 1
1052025-02-21T06:45:13.573Z+ vmstat -T d -p 1
1062025-02-21T06:45:13.576Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1072025-02-21T06:45:13.576Z+ banner test_up_2r_encrypted
1082025-02-21T06:45:13.576Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1092025-02-21T06:45:13.579Z #####
1102025-02-21T06:45:13.579Z ##### ###### #### ##### # # ##### # # #####
1112025-02-21T06:45:13.579Z # # # # # # # # # # #
1122025-02-21T06:45:13.579Z # ##### #### # # # # # ##### # #
1132025-02-21T06:45:13.579Z # # # # # # ##### # #####
1142025-02-21T06:45:13.579Z # # # # # # # # # # #
1152025-02-21T06:45:13.579Z # ###### #### # ####### #### # ####### ####### # #
1162025-02-21T06:45:13.579Z
1172025-02-21T06:45:13.579Z+ ptime -m bash /input/build/work/scripts/test_up.sh -r 2 -N encrypted
1182025-02-21T06:45:13.588Z/input/build/work
1192025-02-21T06:45:13.601ZUsing 2 region sets
1202025-02-21T06:45:13.601ZTurn off color for downstairs dump
1212025-02-21T06:45:13.617ZUpstairs using key: IiYoF+tsmuEzpSI/73MbXasw6mYEBXEhrWPh8OA7ZJI=
1222025-02-21T06:45:13.620Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1232025-02-21T06:45:13.620ZCreating 6 downstairs regions
1242025-02-21T06:45:13.988ZStarting 6 downstairs
1252025-02-21T06:45:13.991Zdsc started at PID: 1198
1262025-02-21T06:45:19.350ZDisable automatic restart on all downstairs
1272025-02-21T06:45:19.406Z
1282025-02-21T06:45:19.406ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1292025-02-21T06:45:19.410ZRunning test: span
1302025-02-21T06:45:23.549ZCompleted test: span
1312025-02-21T06:45:28.555ZRunning test: big
1322025-02-21T06:45:33.147ZCompleted test: big
1332025-02-21T06:45:38.155ZRunning test: dep
1342025-02-21T06:46:11.982ZCompleted test: dep
1352025-02-21T06:46:16.989ZRunning test: balloon
1362025-02-21T06:50:32.714ZCompleted test: balloon
1372025-02-21T06:50:37.721ZRunning test: deactivate
1382025-02-21T06:51:32.377ZCompleted test: deactivate
1392025-02-21T06:51:37.385ZRunning hammer
1402025-02-21T06:51:50.083ZRun repair tests
1412025-02-21T06:51:50.087Z/var/tmp/bins/crutest fill -g 31 -q --key IiYoF+tsmuEzpSI/73MbXasw6mYEBXEhrWPh8OA7ZJI= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1422025-02-21T06:51:50.146Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.581296264Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"task":"crutest"}
1432025-02-21T06:51:50.149Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.581870372Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"task":"crutest"}
1442025-02-21T06:51:50.149Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.58374889Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"task":"crutest"}
1452025-02-21T06:51:50.149Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.58400397Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241}
1462025-02-21T06:51:50.149Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ae555c374784c668297ef5478b8ff4aaa8d57b4\",\n git_commit_timestamp: \"2025-02-21T06:29:49.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4d91de4e48198da2e33413efdcd9cd2cc0c46688\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.584048368Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241}
1472025-02-21T06:51:50.149Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.584074647Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241}
1482025-02-21T06:51:50.149Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.584094787Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241}
1492025-02-21T06:51:50.152Z{"msg":"Crucible 6d054eb5-6c8e-4c72-b704-c435133346f5 has session id: d1073e8f-b764-4250-8b5c-c38495b02018","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.584814299Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1502025-02-21T06:51:50.152Z{"msg":"Upstairs opts: Upstairs UUID: 6d054eb5-6c8e-4c72-b704-c435133346f5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.584879976Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1512025-02-21T06:51:50.152Z{"msg":"Crucible stats registered with UUID: 6d054eb5-6c8e-4c72-b704-c435133346f5","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.584910905Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1522025-02-21T06:51:50.152Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:51:49.584938364Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1532025-02-21T06:51:50.152Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.586984496Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"task":"crutest"}
1542025-02-21T06:51:50.152Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.587065992Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241}
1552025-02-21T06:51:50.152Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ae555c374784c668297ef5478b8ff4aaa8d57b4\",\n git_commit_timestamp: \"2025-02-21T06:29:49.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4d91de4e48198da2e33413efdcd9cd2cc0c46688\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.587109981Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241}
1562025-02-21T06:51:50.152Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.58714143Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241}
1572025-02-21T06:51:50.152Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.587163019Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241}
1582025-02-21T06:51:50.155Z{"msg":"Crucible 6d054eb5-6c8e-4c72-b704-c435133346f5 has session id: 1eae36cf-943f-49d9-a0a8-e07fa3cdf37a","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.587532804Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
1592025-02-21T06:51:50.155Z{"msg":"Upstairs opts: Upstairs UUID: 6d054eb5-6c8e-4c72-b704-c435133346f5, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.587580823Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
1602025-02-21T06:51:50.155Z{"msg":"Crucible stats registered with UUID: 6d054eb5-6c8e-4c72-b704-c435133346f5","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.587609892Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
1612025-02-21T06:51:50.155Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:51:49.587634291Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
1622025-02-21T06:51:50.155Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.588495627Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"task":"crutest"}
1632025-02-21T06:51:50.155Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.588539806Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241}
1642025-02-21T06:51:50.155Z{"msg":"6d054eb5-6c8e-4c72-b704-c435133346f5 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.588649292Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1652025-02-21T06:51:50.155Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.58868952Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"client":"0","":"downstairs","session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1662025-02-21T06:51:50.155Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.588722749Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"client":"0","":"downstairs","session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1672025-02-21T06:51:50.155Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.588752998Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"client":"1","":"downstairs","session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1682025-02-21T06:51:50.155Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.588780096Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"client":"1","":"downstairs","session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1692025-02-21T06:51:50.155Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.588817215Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"client":"2","":"downstairs","session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1702025-02-21T06:51:50.155Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.588844814Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"client":"2","":"downstairs","session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1712025-02-21T06:51:50.155Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.588870573Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"io task","client":"2","":"downstairs","session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1722025-02-21T06:51:50.155Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.588896952Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"io task","client":"0","":"downstairs","session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1732025-02-21T06:51:50.155Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.588932971Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"io task","client":"1","":"downstairs","session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1742025-02-21T06:51:50.155Z{"msg":"ds_connection connected from Ok(127.0.0.1:54532)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.588972439Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"io task","client":"2","":"downstairs","session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1752025-02-21T06:51:50.155Z{"msg":"ds_connection connected from Ok(127.0.0.1:40982)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.589010348Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"io task","client":"0","":"downstairs","session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1762025-02-21T06:51:50.155Z{"msg":"ds_connection connected from Ok(127.0.0.1:57524)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.589238789Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"io task","client":"1","":"downstairs","session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1772025-02-21T06:51:50.155Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.589753839Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"client":"1","":"downstairs","session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1782025-02-21T06:51:50.155Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.589808857Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"client":"2","":"downstairs","session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1792025-02-21T06:51:50.155Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.589839346Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"client":"0","":"downstairs","session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1802025-02-21T06:51:50.159Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.590467432Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"client":"1","":"downstairs","session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1812025-02-21T06:51:50.159Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.590550639Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"client":"0","":"downstairs","session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1822025-02-21T06:51:50.159Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.590595317Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"client":"2","":"downstairs","session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1832025-02-21T06:51:50.159Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.590942413Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1842025-02-21T06:51:50.159Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.591012001Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1852025-02-21T06:51:50.159Z{"msg":"[0]R flush_numbers: [3995, 3995, 3995, 3995, 3995]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.59104686Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"downstairs","":"downstairs","session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1862025-02-21T06:51:50.159Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.591063199Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"downstairs","":"downstairs","session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1872025-02-21T06:51:50.159Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.591078328Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"downstairs","":"downstairs","session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1882025-02-21T06:51:50.159Z{"msg":"[1]R flush_numbers: [3995, 3995, 3995, 3995, 3995]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.591100247Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"downstairs","":"downstairs","session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1892025-02-21T06:51:50.159Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.591114057Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"downstairs","":"downstairs","session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1902025-02-21T06:51:50.160Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.591134466Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"downstairs","":"downstairs","session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1912025-02-21T06:51:50.160Z{"msg":"[2]R flush_numbers: [3995, 3995, 3995, 3995, 3995]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.591148026Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"downstairs","":"downstairs","session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1922025-02-21T06:51:50.160Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.591167795Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"downstairs","":"downstairs","session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1932025-02-21T06:51:50.160Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.591181244Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"downstairs","":"downstairs","session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1942025-02-21T06:51:50.160Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.591200974Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"downstairs","":"downstairs","session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1952025-02-21T06:51:50.160Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.591214163Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"downstairs","":"downstairs","session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1962025-02-21T06:51:50.160Z{"msg":"Next flush: 3996","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.591233602Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"downstairs","":"downstairs","session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1972025-02-21T06:51:50.160Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.591246852Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"downstairs","":"downstairs","session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1982025-02-21T06:51:50.160Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.591273511Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
1992025-02-21T06:51:50.160Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.59129289Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"client":"0","":"downstairs","session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
2002025-02-21T06:51:50.160Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.591327249Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"client":"1","":"downstairs","session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
2012025-02-21T06:51:50.160Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.591353978Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"client":"2","":"downstairs","session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
2022025-02-21T06:51:50.160Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.591376167Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
2032025-02-21T06:51:50.160Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.591398626Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
2042025-02-21T06:51:50.160Z{"msg":"6d054eb5-6c8e-4c72-b704-c435133346f5 is now active with session: d1073e8f-b764-4250-8b5c-c38495b02018","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.591453684Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
2052025-02-21T06:51:50.160Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.591488293Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"session_id":"d1073e8f-b764-4250-8b5c-c38495b02018"}
2062025-02-21T06:51:50.160Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.591695945Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241}
2072025-02-21T06:51:50.160Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.591742833Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241}
2082025-02-21T06:51:50.160Z{"msg":"6d054eb5-6c8e-4c72-b704-c435133346f5 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.591771142Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2092025-02-21T06:51:50.160Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.591841139Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"client":"0","":"downstairs","session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2102025-02-21T06:51:50.160Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.591892807Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"client":"0","":"downstairs","session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2112025-02-21T06:51:50.160Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.591922546Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"client":"1","":"downstairs","session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2122025-02-21T06:51:50.160Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.591948825Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"client":"1","":"downstairs","session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2132025-02-21T06:51:50.160Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.591984504Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"client":"2","":"downstairs","session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2142025-02-21T06:51:50.160Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.592006733Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"client":"2","":"downstairs","session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2152025-02-21T06:51:50.160Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.592033442Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"io task","client":"2","":"downstairs","session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2162025-02-21T06:51:50.160Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.592058661Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"io task","client":"0","":"downstairs","session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2172025-02-21T06:51:50.160Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.59208717Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"io task","client":"1","":"downstairs","session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2182025-02-21T06:51:50.160Z{"msg":"ds_connection connected from Ok(127.0.0.1:56937)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.592110689Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"io task","client":"1","":"downstairs","session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2192025-02-21T06:51:50.160Z{"msg":"ds_connection connected from Ok(127.0.0.1:41830)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.592154607Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"io task","client":"2","":"downstairs","session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2202025-02-21T06:51:50.160Z{"msg":"ds_connection connected from Ok(127.0.0.1:44742)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.592179656Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"io task","client":"0","":"downstairs","session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2212025-02-21T06:51:50.160Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.592195185Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"client":"1","":"downstairs","session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2222025-02-21T06:51:50.160Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.592209455Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"client":"2","":"downstairs","session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2232025-02-21T06:51:50.160Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.592224284Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"client":"0","":"downstairs","session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2242025-02-21T06:51:50.160Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.592556612Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"client":"2","":"downstairs","session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2252025-02-21T06:51:50.160Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.592642608Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"client":"1","":"downstairs","session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2262025-02-21T06:51:50.160Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.592785443Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"client":"0","":"downstairs","session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2272025-02-21T06:51:50.160Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.592960536Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2282025-02-21T06:51:50.161Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.593086041Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2292025-02-21T06:51:50.161Z{"msg":"[0]R flush_numbers: [5316, 5316, 5320, 5315, 5313]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.593184627Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"downstairs","":"downstairs","session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2302025-02-21T06:51:50.161Z{"msg":"[0]R generation: [13, 13, 14, 13, 12]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.593213206Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"downstairs","":"downstairs","session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2312025-02-21T06:51:50.161Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.593228666Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"downstairs","":"downstairs","session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2322025-02-21T06:51:50.161Z{"msg":"[1]R flush_numbers: [5316, 5316, 5320, 5315, 5313]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.593242435Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"downstairs","":"downstairs","session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2332025-02-21T06:51:50.161Z{"msg":"[1]R generation: [13, 13, 14, 13, 12]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.593263704Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"downstairs","":"downstairs","session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2342025-02-21T06:51:50.161Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.593277414Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"downstairs","":"downstairs","session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2352025-02-21T06:51:50.161Z{"msg":"[2]R flush_numbers: [5316, 5316, 5320, 5315, 5313]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
2362025-02-21T06:51:50.161Z,"time":"2025-02-21T06:51:49.593291173Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"downstairs","":"downstairs","session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2372025-02-21T06:51:50.161Z{"msg":"[2]R generation: [13, 13, 14, 13, 12]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.593318462Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"downstairs","":"downstairs","session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2382025-02-21T06:51:50.161Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.593331642Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"downstairs","":"downstairs","session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2392025-02-21T06:51:50.161Z{"msg":"Max found gen is 15","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.593344371Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"downstairs","":"downstairs","session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2402025-02-21T06:51:50.161Z{"msg":"Generation requested: 31 >= found:15","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.593357341Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"downstairs","":"downstairs","session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2412025-02-21T06:51:50.161Z{"msg":"Next flush: 5321","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.59337877Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"downstairs","":"downstairs","session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2422025-02-21T06:51:50.161Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.59339225Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"":"downstairs","":"downstairs","session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2432025-02-21T06:51:50.161Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.593405029Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2442025-02-21T06:51:50.161Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.593424758Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"client":"0","":"downstairs","session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2452025-02-21T06:51:50.161Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.593438738Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"client":"1","":"downstairs","session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2462025-02-21T06:51:50.161Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.593452187Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"client":"2","":"downstairs","session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2472025-02-21T06:51:50.161Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.593472336Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2482025-02-21T06:51:50.161Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.593484596Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2492025-02-21T06:51:50.161Z{"msg":"6d054eb5-6c8e-4c72-b704-c435133346f5 is now active with session: 1eae36cf-943f-49d9-a0a8-e07fa3cdf37a","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.593497086Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2502025-02-21T06:51:50.161Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.593515745Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241,"session_id":"1eae36cf-943f-49d9-a0a8-e07fa3cdf37a"}
2512025-02-21T06:51:50.161Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:49.593527994Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241}
2522025-02-21T06:51:50.161ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
2532025-02-21T06:51:50.161ZFill test
2542025-02-21T06:51:50.200ZRead and Verify all blocks (0..100 range:false)
2552025-02-21T06:51:50.283ZWrote out file "/var/tmp/test_up-build/verify_file"
2562025-02-21T06:51:50.283ZCLIENT: Tests done. All submitted work has been ACK'd
2572025-02-21T06:51:50.286Z----------------------------------------------------------------
2582025-02-21T06:51:50.286Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2592025-02-21T06:51:50.286ZStates: Active Active Active
2602025-02-21T06:51:50.286ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2612025-02-21T06:51:50.286Z 1002 Acked Read 50 Done Done Done false
2622025-02-21T06:51:50.286Z STATES DS:0 DS:1 DS:2 TOTAL
2632025-02-21T06:51:50.286Z Sent 0 0 0 0
2642025-02-21T06:51:50.286Z Done 1 1 1 3
2652025-02-21T06:51:50.286Z Skipped 0 0 0 0
2662025-02-21T06:51:50.287Z Error 0 0 0 0
2672025-02-21T06:51:50.287ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2682025-02-21T06:51:50.287ZDownstairs last five completed: 1001 1000
2692025-02-21T06:51:50.287ZUpstairs last five completed: 1002 1001 1000
2702025-02-21T06:51:50.287Z----------------------------------------------------------------
2712025-02-21T06:51:50.287Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2722025-02-21T06:51:50.287ZStates: Active Active Active
2732025-02-21T06:51:50.287ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2742025-02-21T06:51:50.287Z 1002 Acked Read 50 Done Done Done false
2752025-02-21T06:51:50.287Z STATES DS:0 DS:1 DS:2 TOTAL
2762025-02-21T06:51:50.287Z Sent 0 0 0 0
2772025-02-21T06:51:50.287Z Done 1 1 1 3
2782025-02-21T06:51:50.287Z Skipped 0 0 0 0
2792025-02-21T06:51:50.287Z Error 0 0 0 0
2802025-02-21T06:51:50.287ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2812025-02-21T06:51:50.287ZDownstairs last five completed: 1001 1000
2822025-02-21T06:51:50.287ZUpstairs last five completed: 1002 1001 1000
2832025-02-21T06:51:50.287ZCLIENT: Up:0 ds:2 act:6
2842025-02-21T06:51:54.285Z----------------------------------------------------------------
2852025-02-21T06:51:54.285Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2862025-02-21T06:51:54.285ZDownstairs last five completed: 1003 1002 1001 1000
2872025-02-21T06:51:54.285ZUpstairs last five completed: 1003 1002 1001 1000
2882025-02-21T06:51:54.285Z----------------------------------------------------------------
2892025-02-21T06:51:54.285Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2902025-02-21T06:51:54.285ZDownstairs last five completed: 1003 1002 1001 1000
2912025-02-21T06:51:54.285ZUpstairs last five completed: 1003 1002 1001 1000
2922025-02-21T06:51:54.285ZCLIENT: Up:0 ds:0 act:6
2932025-02-21T06:51:54.285ZCLIENT: All crucible jobs finished, exiting program
2942025-02-21T06:51:54.288Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:51:53.720139438Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1241}
2952025-02-21T06:51:54.296ZRepair setup passed
2962025-02-21T06:51:54.300ZCopy the region for /var/tmp/test_up-build/8830
2972025-02-21T06:51:54.303Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2982025-02-21T06:51:54.305Z/var/tmp/bins/crutest fill -g 32 -q --key IiYoF+tsmuEzpSI/73MbXasw6mYEBXEhrWPh8OA7ZJI= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2992025-02-21T06:51:54.364Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.799081437Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"task":"crutest"}
3002025-02-21T06:51:54.367Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.799674755Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"task":"crutest"}
3012025-02-21T06:51:54.367Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.801575682Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"task":"crutest"}
3022025-02-21T06:51:54.367Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.801813802Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247}
3032025-02-21T06:51:54.367Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ae555c374784c668297ef5478b8ff4aaa8d57b4\",\n git_commit_timestamp: \"2025-02-21T06:29:49.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4d91de4e48198da2e33413efdcd9cd2cc0c46688\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.801860891Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247}
3042025-02-21T06:51:54.367Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.80188777Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247}
3052025-02-21T06:51:54.367Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.801907309Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247}
3062025-02-21T06:51:54.370Z{"msg":"Crucible c23f31ed-e85f-4b46-a91f-cf69936d67ca has session id: 55b7a523-cf45-497f-a067-655f6005474c","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.802737847Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3072025-02-21T06:51:54.370Z{"msg":"Upstairs opts: Upstairs UUID: c23f31ed-e85f-4b46-a91f-cf69936d67ca, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.802812834Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3082025-02-21T06:51:54.370Z{"msg":"Crucible stats registered with UUID: c23f31ed-e85f-4b46-a91f-cf69936d67ca","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.802832143Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3092025-02-21T06:51:54.370Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:51:53.802857762Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3102025-02-21T06:51:54.370Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.804938733Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"task":"crutest"}
3112025-02-21T06:51:54.373Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.805049608Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247}
3122025-02-21T06:51:54.373Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ae555c374784c668297ef5478b8ff4aaa8d57b4\",\n git_commit_timestamp: \"2025-02-21T06:29:49.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4d91de4e48198da2e33413efdcd9cd2cc0c46688\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.805096467Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247}
3132025-02-21T06:51:54.373Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.805113576Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247}
3142025-02-21T06:51:54.373Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.805140685Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247}
3152025-02-21T06:51:54.373Z{"msg":"Crucible c23f31ed-e85f-4b46-a91f-cf69936d67ca has session id: ac8c00fa-b5fe-436d-8bbd-80233662f8c6","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.805553049Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3162025-02-21T06:51:54.373Z{"msg":"Upstairs opts: Upstairs UUID: c23f31ed-e85f-4b46-a91f-cf69936d67ca, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.805615856Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3172025-02-21T06:51:54.373Z{"msg":"Crucible stats registered with UUID: c23f31ed-e85f-4b46-a91f-cf69936d67ca","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.805646875Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3182025-02-21T06:51:54.373Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:51:53.805700603Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3192025-02-21T06:51:54.373Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.806627638Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"task":"crutest"}
3202025-02-21T06:51:54.373Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.806688615Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247}
3212025-02-21T06:51:54.373Z{"msg":"c23f31ed-e85f-4b46-a91f-cf69936d67ca active request set","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.806714594Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3222025-02-21T06:51:54.373Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.806738453Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"client":"0","":"downstairs","session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3232025-02-21T06:51:54.373Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.806775552Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"client":"0","":"downstairs","session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3242025-02-21T06:51:54.373Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.806806721Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"client":"1","":"downstairs","session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3252025-02-21T06:51:54.373Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.806856989Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"client":"1","":"downstairs","session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3262025-02-21T06:51:54.373Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.806885498Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"client":"2","":"downstairs","session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3272025-02-21T06:51:54.373Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.806914617Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"client":"2","":"downstairs","session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3282025-02-21T06:51:54.373Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.806954035Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"io task","client":"2","":"downstairs","session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3292025-02-21T06:51:54.373Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.807003763Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"io task","client":"0","":"downstairs","session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3302025-02-21T06:51:54.373Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.807044792Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"io task","client":"1","":"downstairs","session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3312025-02-21T06:51:54.373Z{"msg":"ds_connection connected from Ok(127.0.0.1:36478)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.807072431Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"io task","client":"1","":"downstairs","session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3322025-02-21T06:51:54.374Z{"msg":"ds_connection connected from Ok(127.0.0.1:42167)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.807112489Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"io task","client":"2","":"downstairs","session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3332025-02-21T06:51:54.374Z{"msg":"ds_connection connected from Ok(127.0.0.1:50504)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.807281183Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"io task","client":"0","":"downstairs","session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3342025-02-21T06:51:54.374Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.807913618Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"client":"0","":"downstairs","session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3352025-02-21T06:51:54.374Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.807962796Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"client":"2","":"downstairs","session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3362025-02-21T06:51:54.374Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.808013124Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"client":"1","":"downstairs","session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3372025-02-21T06:51:54.378Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.808552364Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"client":"1","":"downstairs","session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3382025-02-21T06:51:54.378Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.808755806Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"client":"0","":"downstairs","session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3392025-02-21T06:51:54.378Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.808810484Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"client":"2","":"downstairs","session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3402025-02-21T06:51:54.378Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.809069634Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3412025-02-21T06:51:54.378Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.809117692Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3422025-02-21T06:51:54.378Z{"msg":"[0]R flush_numbers: [3996, 3996, 3996, 3996, 3996]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.809204359Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"downstairs","":"downstairs","session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3432025-02-21T06:51:54.378Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.809241747Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"downstairs","":"downstairs","session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3442025-02-21T06:51:54.378Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.809266696Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"downstairs","":"downstairs","session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3452025-02-21T06:51:54.378Z{"msg":"[1]R flush_numbers: [3996, 3996, 3996, 3996, 3996]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.809296135Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"downstairs","":"downstairs","session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3462025-02-21T06:51:54.378Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.809320484Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"downstairs","":"downstairs","session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3472025-02-21T06:51:54.378Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.809348003Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"downstairs","":"downstairs","session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3482025-02-21T06:51:54.378Z{"msg":"[2]R flush_numbers: [3996, 3996, 3996, 3996, 3996]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.809371652Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"downstairs","":"downstairs","session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3492025-02-21T06:51:54.378Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.809398001Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"downstairs","":"downstairs","session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3502025-02-21T06:51:54.378Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.80942053Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"downstairs","":"downstairs","session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3512025-02-21T06:51:54.378Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.80944372Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"downstairs","":"downstairs","session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3522025-02-21T06:51:54.378Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.809467619Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"downstairs","":"downstairs","session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3532025-02-21T06:51:54.378Z{"msg":"Next flush: 3997","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.809509127Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"downstairs","":"downstairs","session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3542025-02-21T06:51:54.378Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.809532796Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"downstairs","":"downstairs","session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3552025-02-21T06:51:54.378Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.809568475Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3562025-02-21T06:51:54.378Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.809622403Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"client":"0","":"downstairs","session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3572025-02-21T06:51:54.378Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.809650532Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"client":"1","":"downstairs","session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3582025-02-21T06:51:54.378Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.809672801Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"client":"2","":"downstairs","session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3592025-02-21T06:51:54.378Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.809828695Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3602025-02-21T06:51:54.378Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.809857214Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3612025-02-21T06:51:54.378Z{"msg":"c23f31ed-e85f-4b46-a91f-cf69936d67ca is now active with session: 55b7a523-cf45-497f-a067-655f6005474c","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.809905582Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3622025-02-21T06:51:54.379Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.809934261Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"session_id":"55b7a523-cf45-497f-a067-655f6005474c"}
3632025-02-21T06:51:54.379Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.80995922Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247}
3642025-02-21T06:51:54.379Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.809971179Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247}
3652025-02-21T06:51:54.379Z{"msg":"c23f31ed-e85f-4b46-a91f-cf69936d67ca active request set","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.809983109Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3662025-02-21T06:51:54.379Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.810003478Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"client":"0","":"downstairs","session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3672025-02-21T06:51:54.379Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.810028097Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"client":"0","":"downstairs","session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3682025-02-21T06:51:54.379Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.810061486Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"client":"1","":"downstairs","session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3692025-02-21T06:51:54.379Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.810091405Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"client":"1","":"downstairs","session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3702025-02-21T06:51:54.379Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.810128113Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"client":"2","":"downstairs","session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3712025-02-21T06:51:54.379Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.810164372Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"client":"2","":"downstairs","session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3722025-02-21T06:51:54.379Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.810180441Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"io task","client":"2","":"downstairs","session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3732025-02-21T06:51:54.379Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.810201911Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"io task","client":"0","":"downstairs","session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3742025-02-21T06:51:54.379Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.81021614Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"io task","client":"1","":"downstairs","session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3752025-02-21T06:51:54.379Z{"msg":"ds_connection connected from Ok(127.0.0.1:57759)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.810237989Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"io task","client":"2","":"downstairs","session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3762025-02-21T06:51:54.379Z{"msg":"ds_connection connected from Ok(127.0.0.1:34903)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.810262228Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"io task","client":"1","":"downstairs","session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3772025-02-21T06:51:54.379Z{"msg":"ds_connection connected from Ok(127.0.0.1:41785)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.810292027Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"io task","client":"0","":"downstairs","session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3782025-02-21T06:51:54.379Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.810309276Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"client":"1","":"downstairs","session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3792025-02-21T06:51:54.379Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.810322856Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"client":"2","":"downstairs","session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3802025-02-21T06:51:54.379Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.810353345Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"client":"0","":"downstairs","session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3812025-02-21T06:51:54.379Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.810794878Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"client":"2","":"downstairs","session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3822025-02-21T06:51:54.379Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.810869065Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"client":"1","":"downstairs","session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3832025-02-21T06:51:54.379Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.810967161Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"client":"0","":"downstairs","session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3842025-02-21T06:51:54.379Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.811124335Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3852025-02-21T06:51:54.379Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.811229861Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3862025-02-21T06:51:54.379Z{"msg":"[0]R flush_numbers: [5321, 5321, 5321, 5321, 5321]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.811312278Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"downstairs","":"downstairs","session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3872025-02-21T06:51:54.379Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.811348807Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"downstairs","":"downstairs","session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3882025-02-21T06:51:54.379Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.811364766Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"downstairs","":"downstairs","session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3892025-02-21T06:51:54.379Z{"msg":"[1]R flush_numbers: [5321, 5321, 5321, 5321, 5321]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.811387155Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"downstairs","":"downstairs","session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3902025-02-21T06:51:54.379Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.811401004Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"downstairs","":"downstairs","session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3912025-02-21T06:51:54.379Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.811421504Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"downstairs","":"downstairs","session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3922025-02-21T06:51:54.379Z{"msg":"[2]R flush_numbers: [5321, 5321, 5321, 5321, 5321]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.811435883Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"downstairs","":"downstairs","session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3932025-02-21T06:51:54.379Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.811461132Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"downstairs","":"downstairs","session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3942025-02-21T06:51:54.379Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
3952025-02-21T06:51:54.379Z,"time":"2025-02-21T06:51:53.811475171Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"downstairs","":"downstairs","session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3962025-02-21T06:51:54.379Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.81150241Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"downstairs","":"downstairs","session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3972025-02-21T06:51:54.379Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.81151559Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"downstairs","":"downstairs","session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3982025-02-21T06:51:54.379Z{"msg":"Next flush: 5322","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.811535899Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"downstairs","":"downstairs","session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
3992025-02-21T06:51:54.379Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.811549479Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"":"downstairs","":"downstairs","session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
4002025-02-21T06:51:54.379Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.811569258Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
4012025-02-21T06:51:54.380Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.811581487Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"client":"0","":"downstairs","session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
4022025-02-21T06:51:54.380Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.811594917Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"client":"1","":"downstairs","session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
4032025-02-21T06:51:54.380Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.811617166Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"client":"2","":"downstairs","session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
4042025-02-21T06:51:54.380Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.811630645Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
4052025-02-21T06:51:54.380Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.811642715Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
4062025-02-21T06:51:54.380Z{"msg":"c23f31ed-e85f-4b46-a91f-cf69936d67ca is now active with session: ac8c00fa-b5fe-436d-8bbd-80233662f8c6","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.811655425Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
4072025-02-21T06:51:54.380Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.811674104Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247,"session_id":"ac8c00fa-b5fe-436d-8bbd-80233662f8c6"}
4082025-02-21T06:51:54.380Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:53.811686654Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1247}
4092025-02-21T06:51:54.380ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
4102025-02-21T06:51:54.380ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4112025-02-21T06:51:54.380ZFill test
4122025-02-21T06:51:54.420ZRead and Verify all blocks (0..100 range:false)
4132025-02-21T06:51:54.499ZWrote out file "/var/tmp/test_up-build/verify_file"
4142025-02-21T06:51:54.499ZCLIENT: Tests done. All submitted work has been ACK'd
4152025-02-21T06:51:54.502Z----------------------------------------------------------------
4162025-02-21T06:51:54.502Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4172025-02-21T06:51:54.502ZStates: Active Active Active
4182025-02-21T06:51:54.502ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4192025-02-21T06:51:54.502Z 1002 Acked Read 50 Done Done Done false
4202025-02-21T06:51:54.502Z STATES DS:0 DS:1 DS:2 TOTAL
4212025-02-21T06:51:54.502Z Sent 0 0 0 0
4222025-02-21T06:51:54.502Z Done 1 1 1 3
4232025-02-21T06:51:54.502Z Skipped 0 0 0 0
4242025-02-21T06:51:54.502Z Error 0 0 0 0
4252025-02-21T06:51:54.502ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4262025-02-21T06:51:54.502ZDownstairs last five completed: 1001 1000
4272025-02-21T06:51:54.502ZUpstairs last five completed: 1002 1001 1000
4282025-02-21T06:51:54.502Z----------------------------------------------------------------
4292025-02-21T06:51:54.502Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4302025-02-21T06:51:54.502ZStates: Active Active Active
4312025-02-21T06:51:54.502ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4322025-02-21T06:51:54.502Z 1002 Acked Read 50 Done Done Done false
4332025-02-21T06:51:54.502Z STATES DS:0 DS:1 DS:2 TOTAL
4342025-02-21T06:51:54.502Z Sent 0 0 0 0
4352025-02-21T06:51:54.502Z Done 1 1 1 3
4362025-02-21T06:51:54.502Z Skipped 0 0 0 0
4372025-02-21T06:51:54.502Z Error 0 0 0 0
4382025-02-21T06:51:54.502ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4392025-02-21T06:51:54.502ZDownstairs last five completed: 1001 1000
4402025-02-21T06:51:54.502ZUpstairs last five completed: 1002 1001 1000
4412025-02-21T06:51:54.502ZCLIENT: Up:0 ds:2 act:6
4422025-02-21T06:51:58.501Z----------------------------------------------------------------
4432025-02-21T06:51:58.501Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4442025-02-21T06:51:58.501ZDownstairs last five completed: 1003 1002 1001 1000
4452025-02-21T06:51:58.501ZUpstairs last five completed: 1003 1002 1001 1000
4462025-02-21T06:51:58.501Z----------------------------------------------------------------
4472025-02-21T06:51:58.501Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4482025-02-21T06:51:58.501ZDownstairs last five completed: 1003 1002 1001 1000
4492025-02-21T06:51:58.504ZUpstairs last five completed: 1003 1002 1001 1000
4502025-02-21T06:51:58.504ZCLIENT: Up:0 ds:0 act:6
4512025-02-21T06:51:58.505ZCLIENT: All crucible jobs finished, exiting program
4522025-02-21T06:51:58.508ZRepair part 1 passed
4532025-02-21T06:51:58.510Z
4542025-02-21T06:51:58.510ZKill the current downstairs
4552025-02-21T06:51:58.688ZDownstairs 2 stopped
4562025-02-21T06:51:58.689Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
4572025-02-21T06:51:58.691ZNow put back the original so we have a mismatch
4582025-02-21T06:51:58.691Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
4592025-02-21T06:51:58.694ZRestart downstairs with old directory
4602025-02-21T06:51:58.751Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4612025-02-21T06:51:58.769ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4622025-02-21T06:51:58.771Z 0 000-009 32 32 31 3997 3997 3996 F F F <---
4632025-02-21T06:51:58.771Z 1 010-019 32 32 31 3997 3997 3996 F F F <---
4642025-02-21T06:51:58.771Z 2 020-029 32 32 31 3997 3997 3996 F F F <---
4652025-02-21T06:51:58.771Z 3 030-039 32 32 31 3997 3997 3996 F F F <---
4662025-02-21T06:51:58.771Z 4 040-049 32 32 31 3997 3997 3996 F F F <---
4672025-02-21T06:51:58.771ZMax gen: 32, Max flush: 3997
4682025-02-21T06:51:58.771ZError: Difference in extent metadata found!
4692025-02-21T06:51:58.774Zdump test found error as expected
4702025-02-21T06:51:58.774Z
4712025-02-21T06:51:58.774Z
4722025-02-21T06:51:58.774Z/var/tmp/bins/crutest verify --range -g 33 -q --key IiYoF+tsmuEzpSI/73MbXasw6mYEBXEhrWPh8OA7ZJI= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
4732025-02-21T06:51:58.833Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.267836806Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"task":"crutest"}
4742025-02-21T06:51:58.836Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.268409734Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"task":"crutest"}
4752025-02-21T06:51:58.836Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.270124879Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"task":"crutest"}
4762025-02-21T06:51:58.836Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.27034656Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255}
4772025-02-21T06:51:58.836Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ae555c374784c668297ef5478b8ff4aaa8d57b4\",\n git_commit_timestamp: \"2025-02-21T06:29:49.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4d91de4e48198da2e33413efdcd9cd2cc0c46688\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.270383389Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255}
4782025-02-21T06:51:58.836Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.270404658Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255}
4792025-02-21T06:51:58.836Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.270419827Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255}
4802025-02-21T06:51:58.839Z{"msg":"Crucible b8dd2579-0401-434a-905c-d1a3c2e161fe has session id: 1ac77f9c-3e34-4a38-848b-79c22e68f251","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.271226966Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
4812025-02-21T06:51:58.839Z{"msg":"Upstairs opts: Upstairs UUID: b8dd2579-0401-434a-905c-d1a3c2e161fe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.271283494Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
4822025-02-21T06:51:58.839Z{"msg":"Crucible stats registered with UUID: b8dd2579-0401-434a-905c-d1a3c2e161fe","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.271302573Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
4832025-02-21T06:51:58.839Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:51:58.271334832Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
4842025-02-21T06:51:58.839Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.273412392Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"task":"crutest"}
4852025-02-21T06:51:58.839Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.273509669Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255}
4862025-02-21T06:51:58.839Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ae555c374784c668297ef5478b8ff4aaa8d57b4\",\n git_commit_timestamp: \"2025-02-21T06:29:49.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4d91de4e48198da2e33413efdcd9cd2cc0c46688\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.273548907Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255}
4872025-02-21T06:51:58.839Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.273570106Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255}
4882025-02-21T06:51:58.839Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.273585086Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255}
4892025-02-21T06:51:58.843Z{"msg":"Crucible b8dd2579-0401-434a-905c-d1a3c2e161fe has session id: f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.274046268Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
4902025-02-21T06:51:58.843Z{"msg":"Upstairs opts: Upstairs UUID: b8dd2579-0401-434a-905c-d1a3c2e161fe, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.274123465Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
4912025-02-21T06:51:58.843Z{"msg":"Crucible stats registered with UUID: b8dd2579-0401-434a-905c-d1a3c2e161fe","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.274153694Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
4922025-02-21T06:51:58.843Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:51:58.274178423Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
4932025-02-21T06:51:58.843Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.274935114Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"task":"crutest"}
4942025-02-21T06:51:58.843Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.274967653Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255}
4952025-02-21T06:51:58.843Z{"msg":"b8dd2579-0401-434a-905c-d1a3c2e161fe active request set","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.275055059Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
4962025-02-21T06:51:58.843Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.275101628Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"0","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
4972025-02-21T06:51:58.843Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.275147176Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"0","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
4982025-02-21T06:51:58.843Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.275199444Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"1","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
4992025-02-21T06:51:58.843Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.275239992Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"1","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5002025-02-21T06:51:58.843Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.275281581Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"2","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5012025-02-21T06:51:58.843Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.275313019Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"2","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5022025-02-21T06:51:58.843Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.275350348Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5032025-02-21T06:51:58.843Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.275381917Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"io task","client":"0","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5042025-02-21T06:51:58.844Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.275420595Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"io task","client":"1","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5052025-02-21T06:51:58.844Z{"msg":"ds_connection connected from Ok(127.0.0.1:51167)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.275449344Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"io task","client":"1","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5062025-02-21T06:51:58.844Z{"msg":"ds_connection connected from Ok(127.0.0.1:51613)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.275476483Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"io task","client":"0","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5072025-02-21T06:51:58.844Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:51:58.275522601Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5082025-02-21T06:51:58.844Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:51:58.275570299Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5092025-02-21T06:51:58.844Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:51:58.275606548Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5102025-02-21T06:51:58.844Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.275802421Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5112025-02-21T06:51:58.844Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.276285332Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"1","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5122025-02-21T06:51:58.844Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.27632611Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"0","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5132025-02-21T06:51:58.847Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.276955246Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"1","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5142025-02-21T06:51:58.847Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.277007364Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"0","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5152025-02-21T06:51:58.847Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.277407869Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5162025-02-21T06:51:58.847Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:51:58.277452277Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5172025-02-21T06:52:08.843Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.2771169Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5182025-02-21T06:52:08.843Z{"msg":"ds_connection connected from Ok(127.0.0.1:53383)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.277212166Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5192025-02-21T06:52:08.847Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.278210638Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"2","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5202025-02-21T06:52:08.847Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.278797945Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"2","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5212025-02-21T06:52:08.847Z{"msg":"[0]R flush_numbers: [3997, 3997, 3997, 3997, 3997]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.279219859Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"downstairs","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5222025-02-21T06:52:08.847Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.279255068Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"downstairs","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5232025-02-21T06:52:08.848Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.279271157Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"downstairs","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5242025-02-21T06:52:08.848Z{"msg":"[1]R flush_numbers: [3997, 3997, 3997, 3997, 3997]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.279292766Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"downstairs","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5252025-02-21T06:52:08.848Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.279306856Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"downstairs","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5262025-02-21T06:52:08.848Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.279326705Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"downstairs","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5272025-02-21T06:52:08.848Z{"msg":"[2]R flush_numbers: [3996, 3996, 3996, 3996, 3996]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.279340455Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"downstairs","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5282025-02-21T06:52:08.848Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.279360214Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"downstairs","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5292025-02-21T06:52:08.848Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.279374213Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"downstairs","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5302025-02-21T06:52:08.848Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.279387023Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"downstairs","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5312025-02-21T06:52:08.848Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.279407272Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"downstairs","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5322025-02-21T06:52:08.848Z{"msg":"Next flush: 3998","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.279420312Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"downstairs","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5332025-02-21T06:52:08.848Z{"msg":"extent 0 needs reconciliation: ExtentMetadata { gen: 32, flush: 3997, dirty: false } ExtentMetadata { gen: 32, flush: 3997, dirty: false } ExtentMetadata { gen: 31, flush: 3996, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.279442321Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"mend","":"downstairs","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5342025-02-21T06:52:08.848Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 3997, dirty: false }, ExtentMetadata { gen: 32, flush: 3997, dirty: false }, ExtentMetadata { gen: 31, flush: 3996, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.27945876Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"mend","":"downstairs","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5352025-02-21T06:52:08.848Z{"msg":"found dest for source 0 for extent at index 0 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.279480209Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"mend","":"downstairs","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5362025-02-21T06:52:08.848Z{"msg":"extent 1 needs reconciliation: ExtentMetadata { gen: 32, flush: 3997, dirty: false } ExtentMetadata { gen: 32, flush: 3997, dirty: false } ExtentMetadata { gen: 31, flush: 3996, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.279519088Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"mend","":"downstairs","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5372025-02-21T06:52:08.848Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 3997, dirty: false }, ExtentMetadata { gen: 32, flush: 3997, dirty: false }, ExtentMetadata { gen: 31, flush: 3996, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.279545887Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"mend","":"downstairs","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5382025-02-21T06:52:08.848Z{"msg":"found dest for source 0 for extent at index 1 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.279561076Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"mend","":"downstairs","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5392025-02-21T06:52:08.848Z{"msg":"extent 2 needs reconciliation: ExtentMetadata { gen: 32, flush: 3997, dirty: false } ExtentMetadata { gen: 32, flush: 3997, dirty: false } ExtentMetadata { gen: 31, flush: 3996, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.279576375Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"mend","":"downstairs","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5402025-02-21T06:52:08.848Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 3997, dirty: false }, ExtentMetadata { gen: 32, flush: 3997, dirty: false }, ExtentMetadata { gen: 31, flush: 3996, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.279598715Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"mend","":"downstairs","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5412025-02-21T06:52:08.848Z{"msg":"found dest for source 0 for extent at index 2 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.279613144Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"mend","":"downstairs","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5422025-02-21T06:52:08.848Z{"msg":"extent 3 needs reconciliation: ExtentMetadata { gen: 32, flush: 3997, dirty: false } ExtentMetadata { gen: 32, flush: 3997, dirty: false } ExtentMetadata { gen: 31, flush: 3996, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.279639013Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"mend","":"downstairs","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5432025-02-21T06:52:08.848Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 3997, dirty: false }, ExtentMetadata { gen: 32, flush: 3997, dirty: false }, ExtentMetadata { gen: 31, flush: 3996, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.279661942Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"mend","":"downstairs","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5442025-02-21T06:52:08.848Z{"msg":"found dest for source 0 for extent at index 3 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.279675682Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"mend","":"downstairs","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5452025-02-21T06:52:08.848Z{"msg":"extent 4 needs reconciliation: ExtentMetadata { gen: 32, flush: 3997, dirty: false } ExtentMetadata { gen: 32, flush: 3997, dirty: false } ExtentMetadata { gen: 31, flush: 3996, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.279697451Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"mend","":"downstairs","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5462025-02-21T06:52:08.848Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 3997, dirty: false }, ExtentMetadata { gen: 32, flush: 3997, dirty: false }, ExtentMetadata { gen: 31, flush: 3996, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.27971296Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"mend","":"downstairs","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5472025-02-21T06:52:08.848Z{"msg":"found dest for source 0 for extent at index 4 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.27972707Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"mend","":"downstairs","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5482025-02-21T06:52:08.848Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.279753259Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"0","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5492025-02-21T06:52:08.848Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.279780608Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"1","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5502025-02-21T06:52:08.848Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.279821056Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"2","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5512025-02-21T06:52:08.848Z{"msg":"Full repair list: {ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.279857175Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"downstairs","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5522025-02-21T06:52:08.848Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 3998, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 3998, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 3998, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 3998, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 3998, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.280115325Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"downstairs","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5532025-02-21T06:52:08.848Z{"msg":"starting reconciliation e943f8af-14a7-40ad-bcf5-899cf4cbb358: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.280156583Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"downstairs","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5542025-02-21T06:52:08.848Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.280182962Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"2","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5552025-02-21T06:52:08.854Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.288665577Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"2","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5562025-02-21T06:52:08.858Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.292461301Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"2","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5572025-02-21T06:52:08.862Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.296002485Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"2","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5582025-02-21T06:52:08.865Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.29953336Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"2","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5592025-02-21T06:52:08.868Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.302793084Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"downstairs","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5602025-02-21T06:52:08.868Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.302832773Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"0","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5612025-02-21T06:52:08.868Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.302849982Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"1","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5622025-02-21T06:52:08.871Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.302871611Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"2","":"downstairs","session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5632025-02-21T06:52:08.872Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.302885501Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5642025-02-21T06:52:08.872Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.30289804Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5652025-02-21T06:52:08.872Z{"msg":"b8dd2579-0401-434a-905c-d1a3c2e161fe is now active with session: 1ac77f9c-3e34-4a38-848b-79c22e68f251","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.302929949Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"session_id":"1ac77f9c-3e34-4a38-848b-79c22e68f251"}
5662025-02-21T06:52:08.872Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.302984807Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255}
5672025-02-21T06:52:08.872Z{"msg":"b8dd2579-0401-434a-905c-d1a3c2e161fe active request set","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.303004696Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
5682025-02-21T06:52:08.872Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.303018016Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255}
5692025-02-21T06:52:08.872Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.303033065Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"0","":"downstairs","session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
5702025-02-21T06:52:08.872Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.303072144Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"0","":"downstairs","session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
5712025-02-21T06:52:08.872Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.303105482Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"1","":"downstairs","session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
5722025-02-21T06:52:08.872Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.303129751Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"1","":"downstairs","session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
5732025-02-21T06:52:08.872Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.30315349Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"2","":"downstairs","session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
5742025-02-21T06:52:08.872Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.303219338Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"2","":"downstairs","session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
5752025-02-21T06:52:08.872Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.303249867Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
5762025-02-21T06:52:08.872Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.303274116Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"io task","client":"0","":"downstairs","session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
5772025-02-21T06:52:08.872Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.303300305Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"io task","client":"1","":"downstairs","session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
5782025-02-21T06:52:08.872Z{"msg":"ds_connection connected from Ok(127.0.0.1:55924)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.303325504Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
5792025-02-21T06:52:08.872Z{"msg":"ds_connection connected from Ok(127.0.0.1:44079)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.303350703Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"io task","client":"1","":"downstairs","session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
5802025-02-21T06:52:08.872Z{"msg":"ds_connection connected from Ok(127.0.0.1:63865)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.303376032Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"io task","client":"0","":"downstairs","session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
5812025-02-21T06:52:08.872Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.303633582Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"0","":"downstairs","session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
5822025-02-21T06:52:08.872Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.303775726Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"1","":"downstairs","session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
5832025-02-21T06:52:08.872Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.303861863Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"2","":"downstairs","session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
5842025-02-21T06:52:08.872Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.304270458Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"0","":"downstairs","session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
5852025-02-21T06:52:08.872Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.304316456Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"1","":"downstairs","session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
5862025-02-21T06:52:08.872Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.304578736Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"2","":"downstairs","session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
5872025-02-21T06:52:08.872Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.304775168Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
5882025-02-21T06:52:08.872Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.304802737Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
5892025-02-21T06:52:08.872Z{"msg":"[0]R flush_numbers: [5322, 5322, 5322, 5322, 5322]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.304960861Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"downstairs","":"downstairs","session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
5902025-02-21T06:52:08.872Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.30499707Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"downstairs","":"downstairs","session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
5912025-02-21T06:52:08.872Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.305014119Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"downstairs","":"downstairs","session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
5922025-02-21T06:52:08.872Z{"msg":"[1]R flush_numbers: [5322, 5322, 5322, 5322, 5322]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.305028218Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"downstairs","":"downstairs","session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
5932025-02-21T06:52:08.872Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.305041848Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"downstairs","":"downstairs","session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
5942025-02-21T06:52:08.872Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
5952025-02-21T06:52:08.872Z2025-02-21T06:52:08.305063967Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"downstairs","":"downstairs","session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
5962025-02-21T06:52:08.872Z{"msg":"[2]R flush_numbers: [5322, 5322, 5322, 5322, 5322]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.305082866Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"downstairs","":"downstairs","session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
5972025-02-21T06:52:08.872Z{"msg":"[2]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.305097146Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"downstairs","":"downstairs","session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
5982025-02-21T06:52:08.872Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.305110765Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"downstairs","":"downstairs","session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
5992025-02-21T06:52:08.872Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.305124065Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"downstairs","":"downstairs","session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
6002025-02-21T06:52:08.872Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.305136924Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"downstairs","":"downstairs","session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
6012025-02-21T06:52:08.873Z{"msg":"Next flush: 5323","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.305157684Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"downstairs","":"downstairs","session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
6022025-02-21T06:52:08.873Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.305170893Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"":"downstairs","":"downstairs","session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
6032025-02-21T06:52:08.873Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.305183512Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
6042025-02-21T06:52:08.873Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.305202592Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"0","":"downstairs","session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
6052025-02-21T06:52:08.873Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.305215961Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"1","":"downstairs","session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
6062025-02-21T06:52:08.873Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.305230411Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"client":"2","":"downstairs","session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
6072025-02-21T06:52:08.873Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.30524339Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
6082025-02-21T06:52:08.873Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.30526306Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
6092025-02-21T06:52:08.873Z{"msg":"b8dd2579-0401-434a-905c-d1a3c2e161fe is now active with session: f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.305276179Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
6102025-02-21T06:52:08.873Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.305295508Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255,"session_id":"f5c3e7ee-4e29-4286-a27a-64fbb8efbf0a"}
6112025-02-21T06:52:08.873Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:08.305307968Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255}
6122025-02-21T06:52:08.873ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
6132025-02-21T06:52:08.876ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
6142025-02-21T06:52:08.876ZRead and Verify all blocks (0..100 range:true)
6152025-02-21T06:52:08.950ZWrote out file "/var/tmp/test_up-build/verify_file"
6162025-02-21T06:52:08.950ZVerify test completed
6172025-02-21T06:52:08.953ZWrote out file "/var/tmp/test_up-build/verify_file"
6182025-02-21T06:52:08.953ZCLIENT: Tests done. All submitted work has been ACK'd
6192025-02-21T06:52:08.954Z----------------------------------------------------------------
6202025-02-21T06:52:08.954Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6212025-02-21T06:52:08.954ZStates: Active Active Active
6222025-02-21T06:52:08.954ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6232025-02-21T06:52:08.954Z 1000 Acked Read 50 Done Done Done false
6242025-02-21T06:52:08.954Z STATES DS:0 DS:1 DS:2 TOTAL
6252025-02-21T06:52:08.954Z Sent 0 0 0 0
6262025-02-21T06:52:08.954Z Done 1 1 1 3
6272025-02-21T06:52:08.954Z Skipped 0 0 0 0
6282025-02-21T06:52:08.954Z Error 0 0 0 0
6292025-02-21T06:52:08.954ZLast Flush: None None None
6302025-02-21T06:52:08.954ZDownstairs last five completed:
6312025-02-21T06:52:08.954ZUpstairs last five completed: 1000
6322025-02-21T06:52:08.954Z----------------------------------------------------------------
6332025-02-21T06:52:08.954Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6342025-02-21T06:52:08.954ZStates: Active Active Active
6352025-02-21T06:52:08.954ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6362025-02-21T06:52:08.954Z 1000 Acked Read 50 Done Done Done false
6372025-02-21T06:52:08.954Z STATES DS:0 DS:1 DS:2 TOTAL
6382025-02-21T06:52:08.954Z Sent 0 0 0 0
6392025-02-21T06:52:08.954Z Done 1 1 1 3
6402025-02-21T06:52:08.954Z Skipped 0 0 0 0
6412025-02-21T06:52:08.954Z Error 0 0 0 0
6422025-02-21T06:52:08.954ZLast Flush: None None None
6432025-02-21T06:52:08.954ZDownstairs last five completed:
6442025-02-21T06:52:08.954ZUpstairs last five completed: 1000
6452025-02-21T06:52:08.954ZCLIENT: Up:0 ds:2 act:6
6462025-02-21T06:52:12.952Z----------------------------------------------------------------
6472025-02-21T06:52:12.952Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6482025-02-21T06:52:12.952ZDownstairs last five completed: 1001 1000
6492025-02-21T06:52:12.952ZUpstairs last five completed: 1001 1000
6502025-02-21T06:52:12.952Z----------------------------------------------------------------
6512025-02-21T06:52:12.952Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6522025-02-21T06:52:12.952ZDownstairs last five completed: 1001 1000
6532025-02-21T06:52:12.952ZUpstairs last five completed: 1001 1000
6542025-02-21T06:52:12.955ZCLIENT: Up:0 ds:0 act:6
6552025-02-21T06:52:12.955ZCLIENT: All crucible jobs finished, exiting program
6562025-02-21T06:52:12.955Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:12.386433668Z","hostname":"w-01JMKKZ20668FRTV1N5DG1TT8J","pid":1255}
6572025-02-21T06:52:12.960ZRepair part 2 passed
6582025-02-21T06:52:12.962Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
6592025-02-21T06:52:12.978ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6602025-02-21T06:52:12.978Z 0 000-009 32 32 32 3997 3997 3997 F F F
6612025-02-21T06:52:12.979Z 1 010-019 32 32 32 3997 3997 3997 F F F
6622025-02-21T06:52:12.979Z 2 020-029 32 32 32 3997 3997 3997 F F F
6632025-02-21T06:52:12.979Z 3 030-039 32 32 32 3997 3997 3997 F F F
6642025-02-21T06:52:12.979Z 4 040-049 32 32 32 3997 3997 3997 F F F
6652025-02-21T06:52:12.979ZMax gen: 32, Max flush: 3997
6662025-02-21T06:52:12.984Zdump test passed
6672025-02-21T06:52:12.987Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
6682025-02-21T06:52:13.001Z Extent 1
6692025-02-21T06:52:13.004ZGEN 32 32 32
6702025-02-21T06:52:13.004ZFLUSH_ID 3997 3997 3997
6712025-02-21T06:52:13.004ZDIRTY
6722025-02-21T06:52:13.004Z
6732025-02-21T06:52:13.004ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
6742025-02-21T06:52:13.004Z010 A A A A A A
6752025-02-21T06:52:13.007Z011 A A A A A A
6762025-02-21T06:52:13.007Z012 A A A A A A
6772025-02-21T06:52:13.009Z013 A A A A A A
6782025-02-21T06:52:13.009Z014 A A A A A A
6792025-02-21T06:52:13.012Z015 A A A A A A
6802025-02-21T06:52:13.014Z016 A A A A A A
6812025-02-21T06:52:13.014Z017 A A A A A A
6822025-02-21T06:52:13.017Z018 A A A A A A
6832025-02-21T06:52:13.017Z019 A A A A A A
6842025-02-21T06:52:13.022Zdump extent test passed
6852025-02-21T06:52:13.025Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
6862025-02-21T06:52:13.041ZExtent 2 Block in extent 0 Actual block 20
6872025-02-21T06:52:13.041Z
6882025-02-21T06:52:13.044Z DATA SHA256 VER
6892025-02-21T06:52:13.044Z------ ---------------------------------------------------------------- ---
6902025-02-21T06:52:13.044Z 0 b62a9acca299c990ea18d3315e68a5f0bf6c79e0e524ac2f98d8644b0dc2614e A
6912025-02-21T06:52:13.046Z 1 b62a9acca299c990ea18d3315e68a5f0bf6c79e0e524ac2f98d8644b0dc2614e A
6922025-02-21T06:52:13.046Z 2 b62a9acca299c990ea18d3315e68a5f0bf6c79e0e524ac2f98d8644b0dc2614e A
6932025-02-21T06:52:13.046Z
6942025-02-21T06:52:13.046ZNONCES 0 1 2 DIFF
6952025-02-21T06:52:13.046Z------ ------------------------ ------------------------ ------------------------ -----
6962025-02-21T06:52:13.046Z 0 afc75fea89a1137566777e66 afc75fea89a1137566777e66 afc75fea89a1137566777e66
6972025-02-21T06:52:13.046Z
6982025-02-21T06:52:13.046Z TAGS 0 1 2 DIFF
6992025-02-21T06:52:13.046Z------ -------------------------------- -------------------------------- -------------------------------- -----
7002025-02-21T06:52:13.046Z 0 ad518c5224e685dfb05060daaa54e2c4 ad518c5224e685dfb05060daaa54e2c4 ad518c5224e685dfb05060daaa54e2c4
7012025-02-21T06:52:13.046Z
7022025-02-21T06:52:13.047ZHASHES 0 1 2 DIFF
7032025-02-21T06:52:13.047Z------ ---------------- ---------------- ---------------- -----
7042025-02-21T06:52:13.047Z
7052025-02-21T06:52:13.049Zdump block test passed
7062025-02-21T06:52:13.049ZInitial upstairs tests have completed, stopping all downstairs
7072025-02-21T06:52:18.118ZCreating 7 larger downstairs regions
7082025-02-21T06:52:18.303ZStarting 7 downstairs
7092025-02-21T06:52:23.311Zdsc restarted at PID: 1271
7102025-02-21T06:52:23.732ZNow do the replace-reconcile test
7112025-02-21T06:52:23.793ZUsing 8870 for the replacement port
7122025-02-21T06:55:38.351ZCompleted test: replace-reconcile
7132025-02-21T06:55:38.351ZNow do the replace-before-active test
7142025-02-21T06:59:10.778ZCompleted test: replace-before-active
7152025-02-21T06:59:10.781ZAll tests have completed, stopping all downstairs
7162025-02-21T06:59:10.841Z
7172025-02-21T06:59:10.841ZAll Tests have passed
7182025-02-21T06:59:10.844Z13:57 Test duration
7192025-02-21T06:59:10.844Z
7202025-02-21T06:59:10.844Zreal 13:57.212007817
7212025-02-21T06:59:10.844Zuser 18:31.565955677
7222025-02-21T06:59:10.844Zsys 27.340227331
7232025-02-21T06:59:10.844Ztrap 0.567729267
7242025-02-21T06:59:10.844Ztflt 0.044360453
7252025-02-21T06:59:10.844Zdflt 0.049717391
7262025-02-21T06:59:10.844Zkflt 0.001319256
7272025-02-21T06:59:10.844Zlock 25:42:32.728034426
7282025-02-21T06:59:10.845Zslp 1:15:16.672099754
7292025-02-21T06:59:10.845Zlat 1:58.337485078
7302025-02-21T06:59:10.845Zstop 0.214086788
7312025-02-21T06:59:10.845Z+ echo 'test-up-2region-encrypted ends'
7322025-02-21T06:59:10.845Ztest-up-2region-encrypted ends
7332025-02-21T06:59:15.847Zprocess exited: duration 845269 ms, exit code 0
7342025-02-21T06:59:15.847Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
7352025-02-21T07:00:15.880Zexec warning: : stdout descriptor may be held open by a background process; giving up!
7362025-02-21T07:00:15.880Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
7372025-02-21T07:00:15.891Zfound 17 output files
7382025-02-21T07:00:15.891Zuploading: /tmp/test_up-build/dsc-out.txt (53158 bytes)
7392025-02-21T07:00:16.903Zuploaded: /tmp/test_up-build/dsc-out.txt
7402025-02-21T07:00:16.912Zuploading: /tmp/test_up-build/test_up_out.txt (8734449 bytes)
7412025-02-21T07:00:18.041Zuploaded: /tmp/test_up-build/test_up_out.txt
7422025-02-21T07:00:18.044Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (147243 bytes)
7432025-02-21T07:00:19.065Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
7442025-02-21T07:00:19.068Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (144638 bytes)
7452025-02-21T07:00:20.082Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
7462025-02-21T07:00:20.085Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (251058 bytes)
7472025-02-21T07:00:21.109Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
7482025-02-21T07:00:21.112Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (139394 bytes)
7492025-02-21T07:00:22.125Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
7502025-02-21T07:00:22.128Zuploading: /tmp/test_up-build/dsc/downstairs-8850.txt (2494 bytes)
7512025-02-21T07:00:23.133Zuploaded: /tmp/test_up-build/dsc/downstairs-8850.txt
7522025-02-21T07:00:23.136Zuploading: /tmp/test_up-build/dsc/downstairs-8860.txt (36396 bytes)
7532025-02-21T07:00:24.145Zuploaded: /tmp/test_up-build/dsc/downstairs-8860.txt
7542025-02-21T07:00:24.148Zuploading: /tmp/test_up-build/dsc/downstairs-8870.txt (847556 bytes)
7552025-02-21T07:00:25.184Zuploaded: /tmp/test_up-build/dsc/downstairs-8870.txt
7562025-02-21T07:00:25.184Zuploading: /tmp/debug/df.txt (1270 bytes)
7572025-02-21T07:00:26.193Zuploaded: /tmp/debug/df.txt
7582025-02-21T07:00:26.196Zuploading: /tmp/debug/dtrace.txt (1097816 bytes)
7592025-02-21T07:00:27.236Zuploaded: /tmp/debug/dtrace.txt
7602025-02-21T07:00:27.239Zuploading: /tmp/debug/iostat.txt (271878 bytes)
7612025-02-21T07:00:27.255Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 271878 -> 275189
7622025-02-21T07:00:28.261Zuploaded: /tmp/debug/iostat.txt
7632025-02-21T07:00:28.263Zuploading: /tmp/debug/mpstat.txt (659202 bytes)
7642025-02-21T07:00:28.288Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 659202 -> 667962
7652025-02-21T07:00:29.296Zuploaded: /tmp/debug/mpstat.txt
7662025-02-21T07:00:29.299Zuploading: /tmp/debug/paging.txt (119126 bytes)
7672025-02-21T07:00:29.307Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 119126 -> 120792
7682025-02-21T07:00:30.313Zuploaded: /tmp/debug/paging.txt
7692025-02-21T07:00:30.313Zuploading: /tmp/debug/prstat.txt (1284512 bytes)
7702025-02-21T07:00:30.355Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 1284512 -> 1303916
7712025-02-21T07:00:31.361Zuploaded: /tmp/debug/prstat.txt
7722025-02-21T07:00:31.365Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
7732025-02-21T07:00:32.369Zuploaded: /tmp/debug/psrinfo.txt
7742025-02-21T07:00:32.369Zuploading: /tmp/debug/upstairs-info.txt (185130 bytes)
7752025-02-21T07:00:33.381Zuploaded: /tmp/debug/upstairs-info.txt