01JKGZGEACQ9YYK2HV92QRHDS4: test-up-2region-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01JKGZGRMEDJ33WBH34EYYAFSW

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12025-02-07T20:05:42.549Zjob dependencies complete; ready to run (waiting for 13 m 23 s)
22025-02-07T20:06:45.766Zjob assigned to worker 01JKH09JFZ9DAATX9TGCC2XDJM [factory aws, i-0a39c253e7f28f39c] (queued for 1 m 3 s)
32025-02-07T20:06:54.366Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42025-02-07T20:06:56.050Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52025-02-07T20:06:56.050Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62025-02-07T20:06:57.563Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72025-02-07T20:06:57.564Zdownloading input: /input/build/work/bins/crutest.gz
82025-02-07T20:06:59.048Zdownloaded input: /input/build/work/bins/crutest.gz
92025-02-07T20:06:59.048Zdownloading input: /input/build/work/bins/dsc.gz
102025-02-07T20:06:59.930Zdownloaded input: /input/build/work/bins/dsc.gz
112025-02-07T20:06:59.933Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122025-02-07T20:06:59.952Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132025-02-07T20:06:59.952Zdownloading input: /input/build/work/scripts/test_ds.sh
142025-02-07T20:06:59.970Zdownloaded input: /input/build/work/scripts/test_ds.sh
152025-02-07T20:06:59.970Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162025-02-07T20:06:59.990Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172025-02-07T20:06:59.991Zdownloading input: /input/build/work/scripts/test_repair.sh
182025-02-07T20:07:00.010Zdownloaded input: /input/build/work/scripts/test_repair.sh
192025-02-07T20:07:00.010Zdownloading input: /input/build/work/scripts/test_replay.sh
202025-02-07T20:07:00.028Zdownloaded input: /input/build/work/scripts/test_replay.sh
212025-02-07T20:07:00.028Zdownloading input: /input/build/work/scripts/test_up.sh
222025-02-07T20:07:00.047Zdownloaded input: /input/build/work/scripts/test_up.sh
232025-02-07T20:07:00.047Zdownloading input: /input/build/work/scripts/upstairs_info.d
242025-02-07T20:07:00.079Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252025-02-07T20:07:00.082Zdownloading input: /input/build/tmp/cargo-test-out.log
262025-02-07T20:07:00.934Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272025-02-07T20:07:00.935Zstarting task 0: "setup"
282025-02-07T20:07:00.938Z++ uname -s
292025-02-07T20:07:00.938Z+ kern=SunOS
302025-02-07T20:07:00.938Z+ build_user=build
312025-02-07T20:07:00.938Z+ build_uid=12345
322025-02-07T20:07:00.938Z+ work_dir=/work
332025-02-07T20:07:00.938Z+ input_dir=/input
342025-02-07T20:07:00.938Z+ [[ 0 == 12345 ]]
352025-02-07T20:07:00.938Z+ case "$kern" in
362025-02-07T20:07:00.938Z+ groupadd -g 12345 build
372025-02-07T20:07:00.941Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382025-02-07T20:07:02.947Z+ zfs create -o mountpoint=/work rpool/work
392025-02-07T20:07:03.464Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402025-02-07T20:07:03.467Z+ home_fs=zfs
412025-02-07T20:07:03.467Z+ [[ zfs == autofs ]]
422025-02-07T20:07:03.467Z+ mkdir -p /home/build
432025-02-07T20:07:03.469Z+ chown build:build /home/build /work
442025-02-07T20:07:04.471Z+ chmod 0700 /home/build /work
452025-02-07T20:07:04.473Zprocess exited: duration 3545 ms, exit code 0
 
462025-02-07T20:07:04.479Zstarting task 1: "authentication"
472025-02-07T20:07:04.495Zprocess exited: duration 16 ms, exit code 0
 
482025-02-07T20:07:04.501Zstarting task 2: "build"
492025-02-07T20:07:04.506Z+ banner cores
502025-02-07T20:07:04.522Z
512025-02-07T20:07:04.522Z #### #### ##### ###### ####
522025-02-07T20:07:04.522Z # # # # # # # #
532025-02-07T20:07:04.522Z # # # # # ##### ####
542025-02-07T20:07:04.523Z # # # ##### # #
552025-02-07T20:07:04.523Z # # # # # # # # #
562025-02-07T20:07:04.523Z #### #### # # ###### ####
572025-02-07T20:07:04.523Z
582025-02-07T20:07:04.525Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
592025-02-07T20:07:04.531Z+ banner unpack
602025-02-07T20:07:04.533Z
612025-02-07T20:07:04.533Z # # # # ##### ## #### # #
622025-02-07T20:07:04.533Z # # ## # # # # # # # # #
632025-02-07T20:07:04.533Z # # # # # # # # # # ####
642025-02-07T20:07:04.533Z # # # # # ##### ###### # # #
652025-02-07T20:07:04.533Z # # # ## # # # # # # #
662025-02-07T20:07:04.533Z #### # # # # # #### # #
672025-02-07T20:07:04.533Z
682025-02-07T20:07:04.533Z+ mkdir -p /var/tmp/bins
692025-02-07T20:07:04.536Z+ for t in "$input/bins/"*.gz
702025-02-07T20:07:04.536Z++ basename /input/build/work/bins/crucible-downstairs.gz
712025-02-07T20:07:04.539Z+ b=crucible-downstairs.gz
722025-02-07T20:07:04.539Z+ b=crucible-downstairs
732025-02-07T20:07:04.539Z+ gunzip
742025-02-07T20:07:06.933Z+ chmod +x /var/tmp/bins/crucible-downstairs
752025-02-07T20:07:06.936Z+ for t in "$input/bins/"*.gz
762025-02-07T20:07:06.939Z++ basename /input/build/work/bins/crucible-hammer.gz
772025-02-07T20:07:06.939Z+ b=crucible-hammer.gz
782025-02-07T20:07:06.939Z+ b=crucible-hammer
792025-02-07T20:07:06.939Z+ gunzip
802025-02-07T20:07:09.125Z+ chmod +x /var/tmp/bins/crucible-hammer
812025-02-07T20:07:09.128Z+ for t in "$input/bins/"*.gz
822025-02-07T20:07:09.130Z++ basename /input/build/work/bins/crutest.gz
832025-02-07T20:07:09.131Z+ b=crutest.gz
842025-02-07T20:07:09.131Z+ b=crutest
852025-02-07T20:07:09.131Z+ gunzip
862025-02-07T20:07:11.472Z+ chmod +x /var/tmp/bins/crutest
872025-02-07T20:07:11.475Z+ for t in "$input/bins/"*.gz
882025-02-07T20:07:11.477Z++ basename /input/build/work/bins/dsc.gz
892025-02-07T20:07:11.477Z+ b=dsc.gz
902025-02-07T20:07:11.477Z+ b=dsc
912025-02-07T20:07:11.477Z+ gunzip
922025-02-07T20:07:12.892Z+ chmod +x /var/tmp/bins/dsc
932025-02-07T20:07:12.895Z+ export BINDIR=/var/tmp/bins
942025-02-07T20:07:12.895Z+ BINDIR=/var/tmp/bins
952025-02-07T20:07:12.895Z+ jobpid=1150
962025-02-07T20:07:12.895Z+ echo 'Setup debug logging'
972025-02-07T20:07:12.895ZSetup debug logging
982025-02-07T20:07:12.895Z+ mkdir /tmp/debug
992025-02-07T20:07:12.895Z+ sleep 3600
1002025-02-07T20:07:12.898Z+ psrinfo -v
1012025-02-07T20:07:12.900Z+ df -h
1022025-02-07T20:07:12.907Z+ prstat -d d -mLc 1
1032025-02-07T20:07:12.909Z+ iostat -T d -xn 1
1042025-02-07T20:07:12.909Z+ mpstat -T d 1
1052025-02-07T20:07:12.909Z+ vmstat -T d -p 1
1062025-02-07T20:07:12.912Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1072025-02-07T20:07:12.912Z+ banner test_up_2r_encrypted
1082025-02-07T20:07:12.912Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1092025-02-07T20:07:12.914Z #####
1102025-02-07T20:07:12.914Z ##### ###### #### ##### # # ##### # # #####
1112025-02-07T20:07:12.914Z # # # # # # # # # # #
1122025-02-07T20:07:12.914Z # ##### #### # # # # # ##### # #
1132025-02-07T20:07:12.914Z # # # # # # ##### # #####
1142025-02-07T20:07:12.914Z # # # # # # # # # # #
1152025-02-07T20:07:12.915Z # ###### #### # ####### #### # ####### ####### # #
1162025-02-07T20:07:12.915Z
1172025-02-07T20:07:12.915Z+ ptime -m bash /input/build/work/scripts/test_up.sh -r 2 -N encrypted
1182025-02-07T20:07:12.925Z/input/build/work
1192025-02-07T20:07:12.949ZUsing 2 region sets
1202025-02-07T20:07:12.949ZTurn off color for downstairs dump
1212025-02-07T20:07:12.964ZUpstairs using key: jJNBQwSJ9Jj4E3+iMsuKqoCECdaTEMBLNKBOMh2Z6kY=
1222025-02-07T20:07:12.967Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1232025-02-07T20:07:12.967ZCreating 6 downstairs regions
1242025-02-07T20:07:13.324ZStarting 6 downstairs
1252025-02-07T20:07:13.327Zdsc started at PID: 1198
1262025-02-07T20:07:18.683ZDisable automatic restart on all downstairs
1272025-02-07T20:07:18.739Z
1282025-02-07T20:07:18.739ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1292025-02-07T20:07:18.743ZRunning test: span
1302025-02-07T20:07:22.892ZCompleted test: span
1312025-02-07T20:07:27.898ZRunning test: big
1322025-02-07T20:07:32.480ZCompleted test: big
1332025-02-07T20:07:37.486ZRunning test: dep
1342025-02-07T20:08:11.314ZCompleted test: dep
1352025-02-07T20:08:16.320ZRunning test: balloon
1362025-02-07T20:12:32.033ZCompleted test: balloon
1372025-02-07T20:12:37.041ZRunning test: deactivate
1382025-02-07T20:13:31.649ZCompleted test: deactivate
1392025-02-07T20:13:36.656ZRunning hammer
1402025-02-07T20:13:49.582ZRun repair tests
1412025-02-07T20:13:49.585Z/var/tmp/bins/crutest fill -g 31 -q --key jJNBQwSJ9Jj4E3+iMsuKqoCECdaTEMBLNKBOMh2Z6kY= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1422025-02-07T20:13:49.643Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.225516578Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"task":"crutest"}
1432025-02-07T20:13:49.646Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.226060208Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"task":"crutest"}
1442025-02-07T20:13:49.646Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.227637098Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"task":"crutest"}
1452025-02-07T20:13:49.646Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.227833215Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241}
1462025-02-07T20:13:49.646Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"764da36f6a4255a5834591b69c6e5591e6ce3a04\",\n git_commit_timestamp: \"2025-02-07T19:51:46.000000000Z\",\n git_branch: \"alan/agent-needs-workers\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.227879994Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241}
1472025-02-07T20:13:49.646Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.227911013Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241}
1482025-02-07T20:13:49.646Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.227931953Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241}
1492025-02-07T20:13:49.646Z{"msg":"Crucible 4a96880a-84a4-47e7-9214-9ab9fb2431d3 has session id: 2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.228505032Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1502025-02-07T20:13:49.647Z{"msg":"Upstairs opts: Upstairs UUID: 4a96880a-84a4-47e7-9214-9ab9fb2431d3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.228575301Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1512025-02-07T20:13:49.647Z{"msg":"Crucible stats registered with UUID: 4a96880a-84a4-47e7-9214-9ab9fb2431d3","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.22862822Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1522025-02-07T20:13:49.647Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-07T20:13:49.228652939Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1532025-02-07T20:13:49.649Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.230725281Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"task":"crutest"}
1542025-02-07T20:13:49.649Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.230832459Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241}
1552025-02-07T20:13:49.649Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"764da36f6a4255a5834591b69c6e5591e6ce3a04\",\n git_commit_timestamp: \"2025-02-07T19:51:46.000000000Z\",\n git_branch: \"alan/agent-needs-workers\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.230868278Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241}
1562025-02-07T20:13:49.649Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.230888908Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241}
1572025-02-07T20:13:49.650Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.230903267Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241}
1582025-02-07T20:13:49.650Z{"msg":"Crucible 4a96880a-84a4-47e7-9214-9ab9fb2431d3 has session id: b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.2313182Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
1592025-02-07T20:13:49.650Z{"msg":"Upstairs opts: Upstairs UUID: 4a96880a-84a4-47e7-9214-9ab9fb2431d3, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.231373569Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
1602025-02-07T20:13:49.650Z{"msg":"Crucible stats registered with UUID: 4a96880a-84a4-47e7-9214-9ab9fb2431d3","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.231417548Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
1612025-02-07T20:13:49.650Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-07T20:13:49.231441787Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
1622025-02-07T20:13:49.654Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.232230283Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"task":"crutest"}
1632025-02-07T20:13:49.654Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.232261312Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241}
1642025-02-07T20:13:49.654Z{"msg":"4a96880a-84a4-47e7-9214-9ab9fb2431d3 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.23237389Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1652025-02-07T20:13:49.654Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.232406929Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"client":"0","":"downstairs","session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1662025-02-07T20:13:49.654Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.232463538Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"client":"0","":"downstairs","session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1672025-02-07T20:13:49.654Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.232487898Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"client":"1","":"downstairs","session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1682025-02-07T20:13:49.654Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.232510587Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"client":"1","":"downstairs","session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1692025-02-07T20:13:49.654Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.232541197Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"client":"2","":"downstairs","session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1702025-02-07T20:13:49.654Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.232566366Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"client":"2","":"downstairs","session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1712025-02-07T20:13:49.654Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.232593746Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"io task","client":"2","":"downstairs","session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1722025-02-07T20:13:49.654Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.232615166Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"io task","client":"1","":"downstairs","session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1732025-02-07T20:13:49.654Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.232640445Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"io task","client":"0","":"downstairs","session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1742025-02-07T20:13:49.654Z{"msg":"ds_connection connected from Ok(127.0.0.1:62641)","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.232709504Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"io task","client":"2","":"downstairs","session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1752025-02-07T20:13:49.654Z{"msg":"ds_connection connected from Ok(127.0.0.1:40123)","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.232731643Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"io task","client":"1","":"downstairs","session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1762025-02-07T20:13:49.655Z{"msg":"ds_connection connected from Ok(127.0.0.1:64766)","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.232754323Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"io task","client":"0","":"downstairs","session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1772025-02-07T20:13:49.655Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.233403351Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"client":"1","":"downstairs","session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1782025-02-07T20:13:49.655Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.23344835Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"client":"2","":"downstairs","session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1792025-02-07T20:13:49.655Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.233476349Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"client":"0","":"downstairs","session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1802025-02-07T20:13:49.655Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.234085398Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"client":"1","":"downstairs","session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1812025-02-07T20:13:49.655Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.234231415Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"client":"0","":"downstairs","session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1822025-02-07T20:13:49.655Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.234315174Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"client":"2","":"downstairs","session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1832025-02-07T20:13:49.655Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.234476921Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1842025-02-07T20:13:49.655Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.23454283Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1852025-02-07T20:13:49.655Z{"msg":"[0]R flush_numbers: [4000, 4000, 4000, 4000, 4000]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.234652688Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"downstairs","":"downstairs","session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1862025-02-07T20:13:49.655Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.234678247Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"downstairs","":"downstairs","session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1872025-02-07T20:13:49.655Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.234692767Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"downstairs","":"downstairs","session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1882025-02-07T20:13:49.655Z{"msg":"[1]R flush_numbers: [4000, 4000, 4000, 4000, 4000]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.234713496Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"downstairs","":"downstairs","session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1892025-02-07T20:13:49.655Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.234727166Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"downstairs","":"downstairs","session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1902025-02-07T20:13:49.655Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.234746976Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"downstairs","":"downstairs","session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1912025-02-07T20:13:49.655Z{"msg":"[2]R flush_numbers: [4000, 4000, 4000, 4000, 4000]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.234760405Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"downstairs","":"downstairs","session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1922025-02-07T20:13:49.655Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.234779965Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"downstairs","":"downstairs","session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1932025-02-07T20:13:49.655Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.234793075Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"downstairs","":"downstairs","session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1942025-02-07T20:13:49.655Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.234805485Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"downstairs","":"downstairs","session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1952025-02-07T20:13:49.655Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.234825074Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"downstairs","":"downstairs","session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1962025-02-07T20:13:49.655Z{"msg":"Next flush: 4001","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.234838234Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"downstairs","":"downstairs","session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1972025-02-07T20:13:49.655Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.234857234Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"downstairs","":"downstairs","session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1982025-02-07T20:13:49.655Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.234883113Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
1992025-02-07T20:13:49.655Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.234899653Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"client":"0","":"downstairs","session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
2002025-02-07T20:13:49.655Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.234925752Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"client":"1","":"downstairs","session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
2012025-02-07T20:13:49.655Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.234948312Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"client":"2","":"downstairs","session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
2022025-02-07T20:13:49.655Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.234983491Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
2032025-02-07T20:13:49.655Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.235004431Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
2042025-02-07T20:13:49.655Z{"msg":"4a96880a-84a4-47e7-9214-9ab9fb2431d3 is now active with session: 2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.235027141Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
2052025-02-07T20:13:49.655Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.23504954Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"session_id":"2f7d0ba9-58ad-42b4-b6ca-747a3a32be9d"}
2062025-02-07T20:13:49.655Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.235256526Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241}
2072025-02-07T20:13:49.655Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.235304865Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241}
2082025-02-07T20:13:49.655Z{"msg":"4a96880a-84a4-47e7-9214-9ab9fb2431d3 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.235348675Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2092025-02-07T20:13:49.655Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.235380794Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"client":"0","":"downstairs","session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2102025-02-07T20:13:49.655Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.235415593Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"client":"0","":"downstairs","session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2112025-02-07T20:13:49.655Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.235461382Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"client":"1","":"downstairs","session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2122025-02-07T20:13:49.655Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.235501072Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"client":"1","":"downstairs","session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2132025-02-07T20:13:49.655Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.235532041Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"client":"2","":"downstairs","session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2142025-02-07T20:13:49.656Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.235559671Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"client":"2","":"downstairs","session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2152025-02-07T20:13:49.656Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.235648609Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"io task","client":"2","":"downstairs","session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2162025-02-07T20:13:49.659Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.235675078Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"io task","client":"0","":"downstairs","session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2172025-02-07T20:13:49.659Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.235716077Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"io task","client":"1","":"downstairs","session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2182025-02-07T20:13:49.659Z{"msg":"ds_connection connected from Ok(127.0.0.1:64943)","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.235745537Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"io task","client":"2","":"downstairs","session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2192025-02-07T20:13:49.659Z{"msg":"ds_connection connected from Ok(127.0.0.1:46357)","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.235802526Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"io task","client":"1","":"downstairs","session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2202025-02-07T20:13:49.659Z{"msg":"ds_connection connected from Ok(127.0.0.1:32864)","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.235821626Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"io task","client":"0","":"downstairs","session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2212025-02-07T20:13:49.659Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.235841985Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"client":"2","":"downstairs","session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2222025-02-07T20:13:49.659Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.235895084Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"client":"0","":"downstairs","session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2232025-02-07T20:13:49.659Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.235935713Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"client":"1","":"downstairs","session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2242025-02-07T20:13:49.659Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.236418195Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"client":"0","":"downstairs","session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2252025-02-07T20:13:49.659Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.236592671Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"client":"1","":"downstairs","session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2262025-02-07T20:13:49.659Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.236632131Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"client":"2","":"downstairs","session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2272025-02-07T20:13:49.659Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.236810447Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2282025-02-07T20:13:49.659Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.236966294Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2292025-02-07T20:13:49.659Z{"msg":"[0]R flush_numbers: [5317, 5320, 5320, 5320, 5321]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.237052123Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"downstairs","":"downstairs","session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2302025-02-07T20:13:49.659Z{"msg":"[0]R generation: [13, 14, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.237087162Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"downstairs","":"downstairs","session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2312025-02-07T20:13:49.659Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.237103782Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"downstairs","":"downstairs","session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2322025-02-07T20:13:49.659Z{"msg":"[1]R flush_numbers: [5317, 5320, 5320, 5320, 5321]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.237130961Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"downstairs","":"downstairs","session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2332025-02-07T20:13:49.659Z{"msg":"[1]R generation: [13, 14, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.23716933Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"downstairs","":"downstairs","session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2342025-02-07T20:13:49.659Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.23719659Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"downstairs","":"downstairs","session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2352025-02-07T20:13:49.659Z{"msg":"[2]R flush_numbers: [5317, 5320, 5320, 5320, 5321]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.23721379Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"downstairs","":"downstairs","session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2362025-02-07T20:13:49.659Z{"msg":"[2]R generation: [13, 14, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.237249709Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"downstairs","":"downstairs","session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2372025-02-07T20:13:49.659ZWait for a query_work_queue command to finish before sending IO
2382025-02-07T20:13:49.659Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.237289288Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"downstairs","":"downstairs","session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2392025-02-07T20:13:49.659Z{"msg":"Max found gen is 15","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.237324587Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"downstairs","":"downstairs","session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2402025-02-07T20:13:49.660Z{"msg":"Generation requested: 31 >= found:15","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.237360577Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"downstairs","":"downstairs","session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2412025-02-07T20:13:49.660Z{"msg":"Next flush: 5322","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.237384357Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"downstairs","":"downstairs","session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2422025-02-07T20:13:49.660Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.237416976Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"":"downstairs","":"downstairs","session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2432025-02-07T20:13:49.660Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.237443425Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2442025-02-07T20:13:49.660Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.237456045Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"client":"0","":"downstairs","session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2452025-02-07T20:13:49.660Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.237481665Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"client":"1","":"downstairs","session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2462025-02-07T20:13:49.660Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.237507564Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"client":"2","":"downstairs","session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2472025-02-07T20:13:49.660Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.237539963Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2482025-02-07T20:13:49.660Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.237554193Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2492025-02-07T20:13:49.660Z{"msg":"4a96880a-84a4-47e7-9214-9ab9fb2431d3 is now active with session: b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.237579953Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2502025-02-07T20:13:49.660Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.237612632Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241,"session_id":"b7c5ae5e-49d8-4dff-9f02-0b91c54d1fab"}
2512025-02-07T20:13:49.660Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:49.237628122Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1241}
2522025-02-07T20:13:49.660ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
2532025-02-07T20:13:49.660ZFill test
2542025-02-07T20:13:49.698ZRead and Verify all blocks (0..100 range:false)
2552025-02-07T20:13:49.778ZWrote out file "/var/tmp/test_up-build/verify_file"
2562025-02-07T20:13:49.778ZCLIENT: Tests done. All submitted work has been ACK'd
2572025-02-07T20:13:49.778Z----------------------------------------------------------------
2582025-02-07T20:13:49.778Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2592025-02-07T20:13:49.778ZStates: Active Active Active
2602025-02-07T20:13:49.778ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2612025-02-07T20:13:49.781Z 1002 Acked Read 50 Done Done Done false
2622025-02-07T20:13:49.781Z STATES DS:0 DS:1 DS:2 TOTAL
2632025-02-07T20:13:49.781Z Sent 0 0 0 0
2642025-02-07T20:13:49.781Z Done 1 1 1 3
2652025-02-07T20:13:49.781Z Skipped 0 0 0 0
2662025-02-07T20:13:49.781Z Error 0 0 0 0
2672025-02-07T20:13:49.781ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2682025-02-07T20:13:49.781ZDownstairs last five completed: 1001 1000
2692025-02-07T20:13:49.781ZUpstairs last five completed: 1002 1001 1000
2702025-02-07T20:13:49.781Z----------------------------------------------------------------
2712025-02-07T20:13:49.781Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2722025-02-07T20:13:49.781ZStates: Active Active Active
2732025-02-07T20:13:49.781ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2742025-02-07T20:13:49.781Z 1002 Acked Read 50 Done Done Done false
2752025-02-07T20:13:49.781Z STATES DS:0 DS:1 DS:2 TOTAL
2762025-02-07T20:13:49.781Z Sent 0 0 0 0
2772025-02-07T20:13:49.781Z Done 1 1 1 3
2782025-02-07T20:13:49.781Z Skipped 0 0 0 0
2792025-02-07T20:13:49.781Z Error 0 0 0 0
2802025-02-07T20:13:49.781ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2812025-02-07T20:13:49.781ZDownstairs last five completed: 1001 1000
2822025-02-07T20:13:49.781ZUpstairs last five completed: 1002 1001 1000
2832025-02-07T20:13:49.781ZCLIENT: Up:0 ds:2 act:6
2842025-02-07T20:13:53.779Z----------------------------------------------------------------
2852025-02-07T20:13:53.779Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2862025-02-07T20:13:53.779ZDownstairs last five completed: 1003 1002 1001 1000
2872025-02-07T20:13:53.779ZUpstairs last five completed: 1003 1002 1001 1000
2882025-02-07T20:13:53.779Z----------------------------------------------------------------
2892025-02-07T20:13:53.779Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2902025-02-07T20:13:53.779ZDownstairs last five completed: 1003 1002 1001 1000
2912025-02-07T20:13:53.779ZUpstairs last five completed: 1003 1002 1001 1000
2922025-02-07T20:13:53.782ZCLIENT: Up:0 ds:0 act:6
2932025-02-07T20:13:53.782ZCLIENT: All crucible jobs finished, exiting program
2942025-02-07T20:13:53.789ZRepair setup passed
2952025-02-07T20:13:53.793ZCopy the region for /var/tmp/test_up-build/8830
2962025-02-07T20:13:53.796Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2972025-02-07T20:13:53.798Z/var/tmp/bins/crutest fill -g 32 -q --key jJNBQwSJ9Jj4E3+iMsuKqoCECdaTEMBLNKBOMh2Z6kY= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2982025-02-07T20:13:53.855Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.437339469Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"task":"crutest"}
2992025-02-07T20:13:53.858Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.438002286Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"task":"crutest"}
3002025-02-07T20:13:53.858Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.439709514Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"task":"crutest"}
3012025-02-07T20:13:53.858Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.43993301Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247}
3022025-02-07T20:13:53.858Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"764da36f6a4255a5834591b69c6e5591e6ce3a04\",\n git_commit_timestamp: \"2025-02-07T19:51:46.000000000Z\",\n git_branch: \"alan/agent-needs-workers\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.439979089Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247}
3032025-02-07T20:13:53.858Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.440011779Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247}
3042025-02-07T20:13:53.858Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.440033299Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247}
3052025-02-07T20:13:53.861Z{"msg":"Crucible 52036ff4-9d1e-42a3-a63d-a6060dd3b048 has session id: 413ba3d9-10d6-4b70-94e5-e16fedd33382","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.440720176Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3062025-02-07T20:13:53.861Z{"msg":"Upstairs opts: Upstairs UUID: 52036ff4-9d1e-42a3-a63d-a6060dd3b048, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.440759335Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3072025-02-07T20:13:53.861Z{"msg":"Crucible stats registered with UUID: 52036ff4-9d1e-42a3-a63d-a6060dd3b048","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.440782565Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3082025-02-07T20:13:53.861Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-07T20:13:53.440813884Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3092025-02-07T20:13:53.861Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.442821947Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"task":"crutest"}
3102025-02-07T20:13:53.861Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.442906895Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247}
3112025-02-07T20:13:53.861Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"764da36f6a4255a5834591b69c6e5591e6ce3a04\",\n git_commit_timestamp: \"2025-02-07T19:51:46.000000000Z\",\n git_branch: \"alan/agent-needs-workers\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.442942254Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247}
3122025-02-07T20:13:53.861Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.442955574Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247}
3132025-02-07T20:13:53.861Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.442969604Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247}
3142025-02-07T20:13:53.864Z{"msg":"Crucible 52036ff4-9d1e-42a3-a63d-a6060dd3b048 has session id: a3884fab-c9b2-4ca7-b018-666624abf4a9","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.443389326Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3152025-02-07T20:13:53.864Z{"msg":"Upstairs opts: Upstairs UUID: 52036ff4-9d1e-42a3-a63d-a6060dd3b048, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.443443505Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3162025-02-07T20:13:53.864Z{"msg":"Crucible stats registered with UUID: 52036ff4-9d1e-42a3-a63d-a6060dd3b048","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.443471274Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3172025-02-07T20:13:53.864Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-07T20:13:53.443497564Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3182025-02-07T20:13:53.864Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.444470296Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"task":"crutest"}
3192025-02-07T20:13:53.864Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.444538194Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247}
3202025-02-07T20:13:53.864Z{"msg":"52036ff4-9d1e-42a3-a63d-a6060dd3b048 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.444648092Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3212025-02-07T20:13:53.864Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.444707641Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"client":"0","":"downstairs","session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3222025-02-07T20:13:53.864Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.44479829Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"client":"0","":"downstairs","session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3232025-02-07T20:13:53.864Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.444859908Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"client":"1","":"downstairs","session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3242025-02-07T20:13:53.864Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.444913917Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"client":"1","":"downstairs","session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3252025-02-07T20:13:53.864Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.444944097Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"client":"2","":"downstairs","session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3262025-02-07T20:13:53.864Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.444971856Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"client":"2","":"downstairs","session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3272025-02-07T20:13:53.864Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.445061115Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"io task","client":"2","":"downstairs","session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3282025-02-07T20:13:53.864Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.445106174Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"io task","client":"0","":"downstairs","session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3292025-02-07T20:13:53.864Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.445155123Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"io task","client":"1","":"downstairs","session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3302025-02-07T20:13:53.864Z{"msg":"ds_connection connected from Ok(127.0.0.1:49055)","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.445186322Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"io task","client":"0","":"downstairs","session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3312025-02-07T20:13:53.864Z{"msg":"ds_connection connected from Ok(127.0.0.1:56601)","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.445218432Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"io task","client":"1","":"downstairs","session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3322025-02-07T20:13:53.864Z{"msg":"ds_connection connected from Ok(127.0.0.1:64651)","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.445466637Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"io task","client":"2","":"downstairs","session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3332025-02-07T20:13:53.864Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.446039206Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"client":"0","":"downstairs","session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3342025-02-07T20:13:53.864Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.446083326Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"client":"1","":"downstairs","session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3352025-02-07T20:13:53.864Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.446099815Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"client":"2","":"downstairs","session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3362025-02-07T20:13:53.868Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.446638565Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"client":"2","":"downstairs","session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3372025-02-07T20:13:53.868Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.446820082Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"client":"0","":"downstairs","session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3382025-02-07T20:13:53.868Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.446863071Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"client":"1","":"downstairs","session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3392025-02-07T20:13:53.868Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.447128516Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3402025-02-07T20:13:53.868Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.447165615Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3412025-02-07T20:13:53.868Z{"msg":"[0]R flush_numbers: [4001, 4001, 4001, 4001, 4001]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.447258714Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"downstairs","":"downstairs","session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3422025-02-07T20:13:53.868Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.447285453Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"downstairs","":"downstairs","session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3432025-02-07T20:13:53.868Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.447300493Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"downstairs","":"downstairs","session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3442025-02-07T20:13:53.868Z{"msg":"[1]R flush_numbers: [4001, 4001, 4001, 4001, 4001]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.447314163Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"downstairs","":"downstairs","session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3452025-02-07T20:13:53.869Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.447349782Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"downstairs","":"downstairs","session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3462025-02-07T20:13:53.869Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.447379921Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"downstairs","":"downstairs","session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3472025-02-07T20:13:53.869Z{"msg":"[2]R flush_numbers: [4001, 4001, 4001, 4001, 4001]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.447397861Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"downstairs","":"downstairs","session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3482025-02-07T20:13:53.869Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.447428021Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"downstairs","":"downstairs","session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3492025-02-07T20:13:53.869Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.44746523Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"downstairs","":"downstairs","session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3502025-02-07T20:13:53.869Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.447491529Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"downstairs","":"downstairs","session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3512025-02-07T20:13:53.869Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.447505799Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"downstairs","":"downstairs","session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3522025-02-07T20:13:53.869Z{"msg":"Next flush: 4002","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.447535879Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"downstairs","":"downstairs","session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3532025-02-07T20:13:53.869Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.447555288Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"downstairs","":"downstairs","session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3542025-02-07T20:13:53.869Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.447583147Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3552025-02-07T20:13:53.869Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.447595947Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"client":"0","":"downstairs","session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3562025-02-07T20:13:53.869Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.447615087Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"client":"1","":"downstairs","session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3572025-02-07T20:13:53.869Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.447652596Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"client":"2","":"downstairs","session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3582025-02-07T20:13:53.869Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.447676906Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3592025-02-07T20:13:53.869Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.447708485Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3602025-02-07T20:13:53.869Z{"msg":"52036ff4-9d1e-42a3-a63d-a6060dd3b048 is now active with session: 413ba3d9-10d6-4b70-94e5-e16fedd33382","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.447739425Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3612025-02-07T20:13:53.869Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.447760514Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"session_id":"413ba3d9-10d6-4b70-94e5-e16fedd33382"}
3622025-02-07T20:13:53.869Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.447859273Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247}
3632025-02-07T20:13:53.869Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.447912901Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247}
3642025-02-07T20:13:53.869Z{"msg":"52036ff4-9d1e-42a3-a63d-a6060dd3b048 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.447934881Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3652025-02-07T20:13:53.869Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.4479738Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"client":"0","":"downstairs","session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3662025-02-07T20:13:53.869Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.4480049Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"client":"0","":"downstairs","session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3672025-02-07T20:13:53.869Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.448028789Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"client":"1","":"downstairs","session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3682025-02-07T20:13:53.869Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.448055589Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"client":"1","":"downstairs","session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3692025-02-07T20:13:53.869Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.448083568Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"client":"2","":"downstairs","session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3702025-02-07T20:13:53.869Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.448102928Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"client":"2","":"downstairs","session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3712025-02-07T20:13:53.869Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.448126788Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"io task","client":"2","":"downstairs","session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3722025-02-07T20:13:53.869Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.448163777Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"io task","client":"0","":"downstairs","session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3732025-02-07T20:13:53.869Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.448189586Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"io task","client":"1","":"downstairs","session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3742025-02-07T20:13:53.869Z{"msg":"ds_connection connected from Ok(127.0.0.1:60376)","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.448222236Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"io task","client":"0","":"downstairs","session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3752025-02-07T20:13:53.869Z{"msg":"ds_connection connected from Ok(127.0.0.1:56069)","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.448238725Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"io task","client":"2","":"downstairs","session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3762025-02-07T20:13:53.869Z{"msg":"ds_connection connected from Ok(127.0.0.1:42183)","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.448273755Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"io task","client":"1","":"downstairs","session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3772025-02-07T20:13:53.869Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.448346083Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"client":"1","":"downstairs","session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3782025-02-07T20:13:53.869Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.448372353Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"client":"2","":"downstairs","session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3792025-02-07T20:13:53.869Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.448387152Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"client":"0","":"downstairs","session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3802025-02-07T20:13:53.869Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.448937472Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"client":"1","":"downstairs","session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3812025-02-07T20:13:53.869Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.448992821Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"client":"0","":"downstairs","session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3822025-02-07T20:13:53.869Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.449017231Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"client":"2","":"downstairs","session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3832025-02-07T20:13:53.869Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.449322125Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3842025-02-07T20:13:53.870Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.449503202Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3852025-02-07T20:13:53.870Z{"msg":"[0]R flush_numbers: [5322, 5322, 5322, 5322, 5322]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.449535441Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"downstairs","":"downstairs","session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3862025-02-07T20:13:53.870Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.449550911Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"downstairs","":"downstairs","session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3872025-02-07T20:13:53.870Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.44958247Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"downstairs","":"downstairs","session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3882025-02-07T20:13:53.870Z{"msg":"[1]R flush_numbers: [5322, 5322, 5322, 5322, 5322]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.44961528Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"downstairs","":"downstairs","session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3892025-02-07T20:13:53.870Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.449648289Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"downstairs","":"downstairs","session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3902025-02-07T20:13:53.870ZWait for a query_work_queue command to finish before sending IO
3912025-02-07T20:13:53.870Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.449683368Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"downstairs","":"downstairs","session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3922025-02-07T20:13:53.870Z{"msg":"[2]R flush_numbers: [5322, 5322, 5322, 5322, 5322]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.449714848Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"downstairs","":"downstairs","session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3932025-02-07T20:13:53.870Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.449743587Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"downstairs","":"downstairs","session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3942025-02-07T20:13:53.870Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.449776347Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"downstairs","":"downstairs","session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3952025-02-07T20:13:53.870Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.449806236Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"downstairs","":"downstairs","session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3962025-02-07T20:13:53.870Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.449821956Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"downstairs","":"downstairs","session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3972025-02-07T20:13:53.870Z{"msg":"Next flush: 5323","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.449836225Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"downstairs","":"downstairs","session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3982025-02-07T20:13:53.870Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.449858685Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"":"downstairs","":"downstairs","session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
3992025-02-07T20:13:53.870Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.449873175Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
4002025-02-07T20:13:53.870Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.449884795Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"client":"0","":"downstairs","session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
4012025-02-07T20:13:53.873Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.449901484Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"client":"1","":"downstairs","session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
4022025-02-07T20:13:53.873Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.449936584Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"client":"2","":"downstairs","session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
4032025-02-07T20:13:53.873Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.449956433Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
4042025-02-07T20:13:53.873Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.449985453Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
4052025-02-07T20:13:53.873Z{"msg":"52036ff4-9d1e-42a3-a63d-a6060dd3b048 is now active with session: a3884fab-c9b2-4ca7-b018-666624abf4a9","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.450026392Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
4062025-02-07T20:13:53.873Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.450039252Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247,"session_id":"a3884fab-c9b2-4ca7-b018-666624abf4a9"}
4072025-02-07T20:13:53.873Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:53.450057991Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1247}
4082025-02-07T20:13:53.873ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
4092025-02-07T20:13:53.873ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4102025-02-07T20:13:53.873ZFill test
4112025-02-07T20:13:53.909ZRead and Verify all blocks (0..100 range:false)
4122025-02-07T20:13:53.989ZWrote out file "/var/tmp/test_up-build/verify_file"
4132025-02-07T20:13:53.989ZCLIENT: Tests done. All submitted work has been ACK'd
4142025-02-07T20:13:53.992Z----------------------------------------------------------------
4152025-02-07T20:13:53.992Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4162025-02-07T20:13:53.992ZStates: Active Active Active
4172025-02-07T20:13:53.992ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4182025-02-07T20:13:53.992Z 1002 Acked Read 50 Done Done Done false
4192025-02-07T20:13:53.992Z STATES DS:0 DS:1 DS:2 TOTAL
4202025-02-07T20:13:53.992Z Sent 0 0 0 0
4212025-02-07T20:13:53.992Z Done 1 1 1 3
4222025-02-07T20:13:53.992Z Skipped 0 0 0 0
4232025-02-07T20:13:53.992Z Error 0 0 0 0
4242025-02-07T20:13:53.992ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4252025-02-07T20:13:53.992ZDownstairs last five completed: 1001 1000
4262025-02-07T20:13:53.992ZUpstairs last five completed: 1002 1001 1000
4272025-02-07T20:13:53.992Z----------------------------------------------------------------
4282025-02-07T20:13:53.992Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4292025-02-07T20:13:53.992ZStates: Active Active Active
4302025-02-07T20:13:53.992ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4312025-02-07T20:13:53.992Z 1002 Acked Read 50 Done Done Done false
4322025-02-07T20:13:53.992Z STATES DS:0 DS:1 DS:2 TOTAL
4332025-02-07T20:13:53.992Z Sent 0 0 0 0
4342025-02-07T20:13:53.992Z Done 1 1 1 3
4352025-02-07T20:13:53.992Z Skipped 0 0 0 0
4362025-02-07T20:13:53.992Z Error 0 0 0 0
4372025-02-07T20:13:53.992ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4382025-02-07T20:13:53.992ZDownstairs last five completed: 1001 1000
4392025-02-07T20:13:53.993ZUpstairs last five completed: 1002 1001 1000
4402025-02-07T20:13:53.993ZCLIENT: Up:0 ds:2 act:6
4412025-02-07T20:13:57.990Z----------------------------------------------------------------
4422025-02-07T20:13:57.991Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4432025-02-07T20:13:57.991ZDownstairs last five completed: 1003 1002 1001 1000
4442025-02-07T20:13:57.991ZUpstairs last five completed: 1003 1002 1001 1000
4452025-02-07T20:13:57.991Z----------------------------------------------------------------
4462025-02-07T20:13:57.991Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4472025-02-07T20:13:57.991ZDownstairs last five completed: 1003 1002 1001 1000
4482025-02-07T20:13:57.991ZUpstairs last five completed: 1003 1002 1001 1000
4492025-02-07T20:13:57.991ZCLIENT: Up:0 ds:0 act:6
4502025-02-07T20:13:57.991ZCLIENT: All crucible jobs finished, exiting program
4512025-02-07T20:13:57.998ZRepair part 1 passed
4522025-02-07T20:13:58.000Z
4532025-02-07T20:13:58.000ZKill the current downstairs
4542025-02-07T20:13:58.168ZDownstairs 2 stopped
4552025-02-07T20:13:58.168Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
4562025-02-07T20:13:58.171ZNow put back the original so we have a mismatch
4572025-02-07T20:13:58.171Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
4582025-02-07T20:13:58.173ZRestart downstairs with old directory
4592025-02-07T20:13:58.227Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4602025-02-07T20:13:58.245ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4612025-02-07T20:13:58.247Z 0 000-009 32 32 31 4002 4002 4001 F F F <---
4622025-02-07T20:13:58.247Z 1 010-019 32 32 31 4002 4002 4001 F F F <---
4632025-02-07T20:13:58.248Z 2 020-029 32 32 31 4002 4002 4001 F F F <---
4642025-02-07T20:13:58.248Z 3 030-039 32 32 31 4002 4002 4001 F F F <---
4652025-02-07T20:13:58.248Z 4 040-049 32 32 31 4002 4002 4001 F F F <---
4662025-02-07T20:13:58.248ZMax gen: 32, Max flush: 4002
4672025-02-07T20:13:58.248ZError: Difference in extent metadata found!
4682025-02-07T20:13:58.250Zdump test found error as expected
4692025-02-07T20:13:58.250Z
4702025-02-07T20:13:58.250Z
4712025-02-07T20:13:58.250Z/var/tmp/bins/crutest verify --range -g 33 -q --key jJNBQwSJ9Jj4E3+iMsuKqoCECdaTEMBLNKBOMh2Z6kY= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
4722025-02-07T20:13:58.307Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.888724308Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"task":"crutest"}
4732025-02-07T20:13:58.310Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.889244708Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"task":"crutest"}
4742025-02-07T20:13:58.310Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.890801119Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"task":"crutest"}
4752025-02-07T20:13:58.310Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.890988585Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255}
4762025-02-07T20:13:58.310Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"764da36f6a4255a5834591b69c6e5591e6ce3a04\",\n git_commit_timestamp: \"2025-02-07T19:51:46.000000000Z\",\n git_branch: \"alan/agent-needs-workers\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.891031405Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255}
4772025-02-07T20:13:58.310Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.891053884Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255}
4782025-02-07T20:13:58.310Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.891068484Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255}
4792025-02-07T20:13:58.310Z{"msg":"Crucible 9e392009-e6e7-4bcb-933c-cd07ebc2b173 has session id: 06722835-40ac-4935-9ae8-74302391bce3","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.891639513Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
4802025-02-07T20:13:58.310Z{"msg":"Upstairs opts: Upstairs UUID: 9e392009-e6e7-4bcb-933c-cd07ebc2b173, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.891693772Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
4812025-02-07T20:13:58.310Z{"msg":"Crucible stats registered with UUID: 9e392009-e6e7-4bcb-933c-cd07ebc2b173","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.891750041Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
4822025-02-07T20:13:58.310Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-07T20:13:57.891772941Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
4832025-02-07T20:13:58.313Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.893790483Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"task":"crutest"}
4842025-02-07T20:13:58.313Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.893882781Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255}
4852025-02-07T20:13:58.313Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"764da36f6a4255a5834591b69c6e5591e6ce3a04\",\n git_commit_timestamp: \"2025-02-07T19:51:46.000000000Z\",\n git_branch: \"alan/agent-needs-workers\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.893925871Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255}
4862025-02-07T20:13:58.313Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.89394785Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255}
4872025-02-07T20:13:58.313Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.89396316Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255}
4882025-02-07T20:13:58.313Z{"msg":"Crucible 9e392009-e6e7-4bcb-933c-cd07ebc2b173 has session id: 09b9304b-c5ff-4781-b9a4-3a35c67d3fee","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.894316453Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
4892025-02-07T20:13:58.313Z{"msg":"Upstairs opts: Upstairs UUID: 9e392009-e6e7-4bcb-933c-cd07ebc2b173, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.894372742Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
4902025-02-07T20:13:58.313Z{"msg":"Crucible stats registered with UUID: 9e392009-e6e7-4bcb-933c-cd07ebc2b173","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.894400482Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
4912025-02-07T20:13:58.313Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-07T20:13:57.894436711Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
4922025-02-07T20:13:58.313Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.895139088Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"task":"crutest"}
4932025-02-07T20:13:58.313Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.895193017Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255}
4942025-02-07T20:13:58.316Z{"msg":"9e392009-e6e7-4bcb-933c-cd07ebc2b173 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.895266786Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
4952025-02-07T20:13:58.316Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.895311115Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"0","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
4962025-02-07T20:13:58.316Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.895355064Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"0","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
4972025-02-07T20:13:58.316Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.895385133Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"1","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
4982025-02-07T20:13:58.316Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.895428912Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"1","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
4992025-02-07T20:13:58.316Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.895469282Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"2","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5002025-02-07T20:13:58.316Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.895497041Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"2","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5012025-02-07T20:13:58.316Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.89554495Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5022025-02-07T20:13:58.316Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.8955876Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"io task","client":"0","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5032025-02-07T20:13:58.316Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.895615079Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"io task","client":"1","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5042025-02-07T20:13:58.316Z{"msg":"ds_connection connected from Ok(127.0.0.1:55717)","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.895650828Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"io task","client":"0","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5052025-02-07T20:13:58.316Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-07T20:13:57.895682258Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5062025-02-07T20:13:58.316Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-07T20:13:57.895724477Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5072025-02-07T20:13:58.316Z{"msg":"ds_connection connected from Ok(127.0.0.1:42868)","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.895751796Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"io task","client":"1","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5082025-02-07T20:13:58.317Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-07T20:13:57.895780666Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5092025-02-07T20:13:58.317Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.895952733Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5102025-02-07T20:13:58.317Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.896293276Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"0","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5112025-02-07T20:13:58.317Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.896329666Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"1","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5122025-02-07T20:13:58.317Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.896933434Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"0","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5132025-02-07T20:13:58.317Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.897058702Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"1","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5142025-02-07T20:13:58.317Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.897302797Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5152025-02-07T20:13:58.317Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:13:57.897427775Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5162025-02-07T20:14:08.316Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.897469385Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5172025-02-07T20:14:08.320Z{"msg":"ds_connection connected from Ok(127.0.0.1:45340)","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.897630342Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5182025-02-07T20:14:08.320Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.898616573Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"2","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5192025-02-07T20:14:08.320Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.89930603Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"2","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5202025-02-07T20:14:08.320Z{"msg":"[0]R flush_numbers: [4002, 4002, 4002, 4002, 4002]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.899752112Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"downstairs","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5212025-02-07T20:14:08.320Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.899797361Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"downstairs","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5222025-02-07T20:14:08.320Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.899815831Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"downstairs","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5232025-02-07T20:14:08.320Z{"msg":"[1]R flush_numbers: [4002, 4002, 4002, 4002, 4002]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.899830161Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"downstairs","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5242025-02-07T20:14:08.320Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.89985754Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"downstairs","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5252025-02-07T20:14:08.320Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.89988102Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"downstairs","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5262025-02-07T20:14:08.320Z{"msg":"[2]R flush_numbers: [4001, 4001, 4001, 4001, 4001]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.899895549Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"downstairs","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5272025-02-07T20:14:08.320Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.899917359Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"downstairs","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5282025-02-07T20:14:08.320Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.899931559Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"downstairs","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5292025-02-07T20:14:08.320Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.899951148Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"downstairs","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5302025-02-07T20:14:08.320Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.899964908Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"downstairs","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5312025-02-07T20:14:08.320Z{"msg":"Next flush: 4003","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.899978158Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"downstairs","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5322025-02-07T20:14:08.321Z{"msg":"extent 0 needs reconciliation: ExtentMetadata { gen: 32, flush: 4002, dirty: false } ExtentMetadata { gen: 32, flush: 4002, dirty: false } ExtentMetadata { gen: 31, flush: 4001, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.899994327Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"mend","":"downstairs","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5332025-02-07T20:14:08.321Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 4002, dirty: false }, ExtentMetadata { gen: 32, flush: 4002, dirty: false }, ExtentMetadata { gen: 31, flush: 4001, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.900017587Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"mend","":"downstairs","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5342025-02-07T20:14:08.321Z{"msg":"found dest for source 0 for extent at index 0 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.900032047Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"mend","":"downstairs","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5352025-02-07T20:14:08.321Z{"msg":"extent 1 needs reconciliation: ExtentMetadata { gen: 32, flush: 4002, dirty: false } ExtentMetadata { gen: 32, flush: 4002, dirty: false } ExtentMetadata { gen: 31, flush: 4001, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.900047286Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"mend","":"downstairs","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5362025-02-07T20:14:08.321Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 4002, dirty: false }, ExtentMetadata { gen: 32, flush: 4002, dirty: false }, ExtentMetadata { gen: 31, flush: 4001, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.900070056Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"mend","":"downstairs","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5372025-02-07T20:14:08.321Z{"msg":"found dest for source 0 for extent at index 1 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.900084156Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"mend","":"downstairs","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5382025-02-07T20:14:08.321Z{"msg":"extent 2 needs reconciliation: ExtentMetadata { gen: 32, flush: 4002, dirty: false } ExtentMetadata { gen: 32, flush: 4002, dirty: false } ExtentMetadata { gen: 31, flush: 4001, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.900111305Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"mend","":"downstairs","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5392025-02-07T20:14:08.321Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 4002, dirty: false }, ExtentMetadata { gen: 32, flush: 4002, dirty: false }, ExtentMetadata { gen: 31, flush: 4001, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.900129655Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"mend","":"downstairs","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5402025-02-07T20:14:08.321Z{"msg":"found dest for source 0 for extent at index 2 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.900154544Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"mend","":"downstairs","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5412025-02-07T20:14:08.321Z{"msg":"extent 3 needs reconciliation: ExtentMetadata { gen: 32, flush: 4002, dirty: false } ExtentMetadata { gen: 32, flush: 4002, dirty: false } ExtentMetadata { gen: 31, flush: 4001, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.900170334Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"mend","":"downstairs","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5422025-02-07T20:14:08.321Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 4002, dirty: false }, ExtentMetadata { gen: 32, flush: 4002, dirty: false }, ExtentMetadata { gen: 31, flush: 4001, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.900195724Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"mend","":"downstairs","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5432025-02-07T20:14:08.321Z{"msg":"found dest for source 0 for extent at index 3 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.900209874Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"mend","":"downstairs","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5442025-02-07T20:14:08.321Z{"msg":"extent 4 needs reconciliation: ExtentMetadata { gen: 32, flush: 4002, dirty: false } ExtentMetadata { gen: 32, flush: 4002, dirty: false } ExtentMetadata { gen: 31, flush: 4001, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.900232123Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"mend","":"downstairs","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5452025-02-07T20:14:08.321Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 4002, dirty: false }, ExtentMetadata { gen: 32, flush: 4002, dirty: false }, ExtentMetadata { gen: 31, flush: 4001, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.900247503Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"mend","":"downstairs","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5462025-02-07T20:14:08.321Z{"msg":"found dest for source 0 for extent at index 4 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.900268372Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"mend","":"downstairs","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5472025-02-07T20:14:08.321Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.900291812Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"0","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5482025-02-07T20:14:08.321Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.900313712Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"1","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5492025-02-07T20:14:08.321Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.900327541Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"2","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5502025-02-07T20:14:08.321Z{"msg":"Full repair list: {ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.900345721Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"downstairs","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5512025-02-07T20:14:08.321Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 4003, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 4003, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 4003, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 4003, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 4003, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.900443799Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"downstairs","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5522025-02-07T20:14:08.321Z{"msg":"starting reconciliation 6fe3a23e-11f3-44ba-853e-87fcfac1a5ac: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.900463729Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"downstairs","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5532025-02-07T20:14:08.324Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.901070107Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"2","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5542025-02-07T20:14:08.328Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.909972171Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"2","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5552025-02-07T20:14:08.332Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.913630333Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"2","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5562025-02-07T20:14:08.335Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.917060759Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"2","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5572025-02-07T20:14:08.339Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.920804179Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"2","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5582025-02-07T20:14:08.342Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.923793863Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"downstairs","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5592025-02-07T20:14:08.342Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.923834562Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"0","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5602025-02-07T20:14:08.342Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.923850562Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"1","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5612025-02-07T20:14:08.342Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.923865222Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"2","":"downstairs","session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5622025-02-07T20:14:08.345Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.923879322Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5632025-02-07T20:14:08.345Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.923891672Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5642025-02-07T20:14:08.345Z{"msg":"9e392009-e6e7-4bcb-933c-cd07ebc2b173 is now active with session: 06722835-40ac-4935-9ae8-74302391bce3","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.923905121Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"session_id":"06722835-40ac-4935-9ae8-74302391bce3"}
5652025-02-07T20:14:08.345Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.923918341Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255}
5662025-02-07T20:14:08.345Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.92394434Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255}
5672025-02-07T20:14:08.345Z{"msg":"9e392009-e6e7-4bcb-933c-cd07ebc2b173 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.92396381Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
5682025-02-07T20:14:08.345Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.92397724Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"0","":"downstairs","session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
5692025-02-07T20:14:08.345Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.92399903Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"0","":"downstairs","session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
5702025-02-07T20:14:08.345Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.924013149Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"1","":"downstairs","session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
5712025-02-07T20:14:08.345Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.924026859Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"1","":"downstairs","session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
5722025-02-07T20:14:08.345Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.924039879Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"2","":"downstairs","session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
5732025-02-07T20:14:08.345Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.924053478Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"2","":"downstairs","session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
5742025-02-07T20:14:08.345Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.924086198Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
5752025-02-07T20:14:08.345Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.924113667Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"io task","client":"0","":"downstairs","session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
5762025-02-07T20:14:08.345Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.924149827Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"io task","client":"1","":"downstairs","session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
5772025-02-07T20:14:08.345Z{"msg":"ds_connection connected from Ok(127.0.0.1:34512)","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.924177866Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
5782025-02-07T20:14:08.345Z{"msg":"ds_connection connected from Ok(127.0.0.1:52983)","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.924204026Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"io task","client":"0","":"downstairs","session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
5792025-02-07T20:14:08.345Z{"msg":"ds_connection connected from Ok(127.0.0.1:43697)","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.924230005Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"io task","client":"1","":"downstairs","session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
5802025-02-07T20:14:08.345Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.924640087Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"2","":"downstairs","session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
5812025-02-07T20:14:08.346Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.924685997Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"0","":"downstairs","session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
5822025-02-07T20:14:08.346Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.924717976Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"1","":"downstairs","session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
5832025-02-07T20:14:08.346Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.925161028Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"1","":"downstairs","session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
5842025-02-07T20:14:08.346Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.925300545Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"2","":"downstairs","session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
5852025-02-07T20:14:08.346Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.925354904Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"0","":"downstairs","session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
5862025-02-07T20:14:08.346Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.925517621Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
5872025-02-07T20:14:08.346Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.925652119Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
5882025-02-07T20:14:08.346Z{"msg":"[0]R flush_numbers: [5323, 5323, 5323, 5323, 5323]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.925745557Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"downstairs","":"downstairs","session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
5892025-02-07T20:14:08.346Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.925773996Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"downstairs","":"downstairs","session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
5902025-02-07T20:14:08.346Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.925791936Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"downstairs","":"downstairs","session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
5912025-02-07T20:14:08.346Z{"msg":"[1]R flush_numbers: [5323, 5323, 5323, 5323, 5323]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.925821996Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"downstairs","":"downstairs","session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
5922025-02-07T20:14:08.346Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.925837555Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"downstairs","":"downstairs","session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
5932025-02-07T20:14:08.346Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
5942025-02-07T20:14:08.346Z,"time":"2025-02-07T20:14:07.925859395Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"downstairs","":"downstairs","session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
5952025-02-07T20:14:08.346Z{"msg":"[2]R flush_numbers: [5323, 5323, 5323, 5323, 5323]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.925889414Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"downstairs","":"downstairs","session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
5962025-02-07T20:14:08.346Z{"msg":"[2]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.925903994Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"downstairs","":"downstairs","session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
5972025-02-07T20:14:08.346Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.925923274Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"downstairs","":"downstairs","session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
5982025-02-07T20:14:08.346Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.925943843Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"downstairs","":"downstairs","session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
5992025-02-07T20:14:08.346Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.925958323Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"downstairs","":"downstairs","session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
6002025-02-07T20:14:08.346Z{"msg":"Next flush: 5324","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.925974063Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"downstairs","":"downstairs","session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
6012025-02-07T20:14:08.346Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.926017092Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"":"downstairs","":"downstairs","session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
6022025-02-07T20:14:08.346Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.926034601Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
6032025-02-07T20:14:08.346Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.926066191Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"0","":"downstairs","session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
6042025-02-07T20:14:08.346Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.92609902Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"1","":"downstairs","session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
6052025-02-07T20:14:08.346Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.92613325Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"client":"2","":"downstairs","session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
6062025-02-07T20:14:08.346Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.926152779Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
6072025-02-07T20:14:08.346Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.926169239Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
6082025-02-07T20:14:08.346Z{"msg":"9e392009-e6e7-4bcb-933c-cd07ebc2b173 is now active with session: 09b9304b-c5ff-4781-b9a4-3a35c67d3fee","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.926197409Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
6092025-02-07T20:14:08.346Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.926210698Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255,"session_id":"09b9304b-c5ff-4781-b9a4-3a35c67d3fee"}
6102025-02-07T20:14:08.346Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-02-07T20:14:07.926222678Z","hostname":"w-01JKH09JFZ9DAATX9TGCC2XDJM","pid":1255}
6112025-02-07T20:14:08.346ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
6122025-02-07T20:14:08.346ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
6132025-02-07T20:14:08.346ZRead and Verify all blocks (0..100 range:true)
6142025-02-07T20:14:08.425ZWrote out file "/var/tmp/test_up-build/verify_file"
6152025-02-07T20:14:08.425ZVerify test completed
6162025-02-07T20:14:08.428ZWrote out file "/var/tmp/test_up-build/verify_file"
6172025-02-07T20:14:08.428ZCLIENT: Tests done. All submitted work has been ACK'd
6182025-02-07T20:14:08.428Z----------------------------------------------------------------
6192025-02-07T20:14:08.428Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6202025-02-07T20:14:08.428ZStates: Active Active Active
6212025-02-07T20:14:08.428ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6222025-02-07T20:14:08.428Z 1000 Acked Read 50 Done Done Done false
6232025-02-07T20:14:08.428Z STATES DS:0 DS:1 DS:2 TOTAL
6242025-02-07T20:14:08.428Z Sent 0 0 0 0
6252025-02-07T20:14:08.428Z Done 1 1 1 3
6262025-02-07T20:14:08.428Z Skipped 0 0 0 0
6272025-02-07T20:14:08.428Z Error 0 0 0 0
6282025-02-07T20:14:08.428ZLast Flush: None None None
6292025-02-07T20:14:08.428ZDownstairs last five completed:
6302025-02-07T20:14:08.428ZUpstairs last five completed: 1000
6312025-02-07T20:14:08.428Z----------------------------------------------------------------
6322025-02-07T20:14:08.428Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6332025-02-07T20:14:08.428ZStates: Active Active Active
6342025-02-07T20:14:08.428ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6352025-02-07T20:14:08.428Z 1000 Acked Read 50 Done Done Done false
6362025-02-07T20:14:08.428Z STATES DS:0 DS:1 DS:2 TOTAL
6372025-02-07T20:14:08.428Z Sent 0 0 0 0
6382025-02-07T20:14:08.428Z Done 1 1 1 3
6392025-02-07T20:14:08.428Z Skipped 0 0 0 0
6402025-02-07T20:14:08.429Z Error 0 0 0 0
6412025-02-07T20:14:08.429ZLast Flush: None None None
6422025-02-07T20:14:08.429ZDownstairs last five completed:
6432025-02-07T20:14:08.429ZUpstairs last five completed: 1000
6442025-02-07T20:14:08.429ZCLIENT: Up:0 ds:2 act:6
6452025-02-07T20:14:12.426Z----------------------------------------------------------------
6462025-02-07T20:14:12.426Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6472025-02-07T20:14:12.426ZDownstairs last five completed: 1001 1000
6482025-02-07T20:14:12.426ZUpstairs last five completed: 1001 1000
6492025-02-07T20:14:12.426Z----------------------------------------------------------------
6502025-02-07T20:14:12.426Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6512025-02-07T20:14:12.426ZDownstairs last five completed: 1001 1000
6522025-02-07T20:14:12.426ZUpstairs last five completed: 1001 1000
6532025-02-07T20:14:12.426ZCLIENT: Up:0 ds:0 act:6
6542025-02-07T20:14:12.426ZCLIENT: All crucible jobs finished, exiting program
6552025-02-07T20:14:12.434ZRepair part 2 passed
6562025-02-07T20:14:12.436Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
6572025-02-07T20:14:12.453ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6582025-02-07T20:14:12.453Z 0 000-009 32 32 32 4002 4002 4002 F F F
6592025-02-07T20:14:12.453Z 1 010-019 32 32 32 4002 4002 4002 F F F
6602025-02-07T20:14:12.453Z 2 020-029 32 32 32 4002 4002 4002 F F F
6612025-02-07T20:14:12.453Z 3 030-039 32 32 32 4002 4002 4002 F F F
6622025-02-07T20:14:12.453Z 4 040-049 32 32 32 4002 4002 4002 F F F
6632025-02-07T20:14:12.453ZMax gen: 32, Max flush: 4002
6642025-02-07T20:14:12.457Zdump test passed
6652025-02-07T20:14:12.460Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
6662025-02-07T20:14:12.474Z Extent 1
6672025-02-07T20:14:12.477ZGEN 32 32 32
6682025-02-07T20:14:12.477ZFLUSH_ID 4002 4002 4002
6692025-02-07T20:14:12.477ZDIRTY
6702025-02-07T20:14:12.477Z
6712025-02-07T20:14:12.477ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
6722025-02-07T20:14:12.477Z010 A A A A A A
6732025-02-07T20:14:12.479Z011 A A A A A A
6742025-02-07T20:14:12.479Z012 A A A A A A
6752025-02-07T20:14:12.482Z013 A A A A A A
6762025-02-07T20:14:12.482Z014 A A A A A A
6772025-02-07T20:14:12.485Z015 A A A A A A
6782025-02-07T20:14:12.485Z016 A A A A A A
6792025-02-07T20:14:12.487Z017 A A A A A A
6802025-02-07T20:14:12.490Z018 A A A A A A
6812025-02-07T20:14:12.490Z019 A A A A A A
6822025-02-07T20:14:12.494Zdump extent test passed
6832025-02-07T20:14:12.496Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
6842025-02-07T20:14:12.510ZExtent 2 Block in extent 0 Actual block 20
6852025-02-07T20:14:12.510Z
6862025-02-07T20:14:12.514Z DATA SHA256 VER
6872025-02-07T20:14:12.514Z------ ---------------------------------------------------------------- ---
6882025-02-07T20:14:12.514Z 0 0332eca356c276e06b79716631a7f03085a966c1f0993d246463e66b511a3912 A
6892025-02-07T20:14:12.516Z 1 0332eca356c276e06b79716631a7f03085a966c1f0993d246463e66b511a3912 A
6902025-02-07T20:14:12.516Z 2 0332eca356c276e06b79716631a7f03085a966c1f0993d246463e66b511a3912 A
6912025-02-07T20:14:12.516Z
6922025-02-07T20:14:12.516ZNONCES 0 1 2 DIFF
6932025-02-07T20:14:12.516Z------ ------------------------ ------------------------ ------------------------ -----
6942025-02-07T20:14:12.516Z 0 36047a04febdd9bebe42a9a4 36047a04febdd9bebe42a9a4 36047a04febdd9bebe42a9a4
6952025-02-07T20:14:12.516Z
6962025-02-07T20:14:12.517Z TAGS 0 1 2 DIFF
6972025-02-07T20:14:12.517Z------ -------------------------------- -------------------------------- -------------------------------- -----
6982025-02-07T20:14:12.517Z 0 f6aac4a639685e834226895ead7aef43 f6aac4a639685e834226895ead7aef43 f6aac4a639685e834226895ead7aef43
6992025-02-07T20:14:12.517Z
7002025-02-07T20:14:12.517ZHASHES 0 1 2 DIFF
7012025-02-07T20:14:12.517Z------ ---------------- ---------------- ---------------- -----
7022025-02-07T20:14:12.517Z
7032025-02-07T20:14:12.519Zdump block test passed
7042025-02-07T20:14:12.519ZInitial upstairs tests have completed, stopping all downstairs
7052025-02-07T20:14:17.582ZCreating 7 larger downstairs regions
7062025-02-07T20:14:17.776ZStarting 7 downstairs
7072025-02-07T20:14:22.784Zdsc restarted at PID: 1271
7082025-02-07T20:14:23.178ZNow do the replace-reconcile test
7092025-02-07T20:14:23.235ZUsing 8870 for the replacement port
7102025-02-07T20:17:00.573ZCompleted test: replace-reconcile
7112025-02-07T20:17:00.574ZNow do the replace-before-active test
7122025-02-07T20:20:34.631ZCompleted test: replace-before-active
7132025-02-07T20:20:34.631ZAll tests have completed, stopping all downstairs
7142025-02-07T20:20:34.692Z
7152025-02-07T20:20:34.692ZAll Tests have passed
7162025-02-07T20:20:34.695Z13:22 Test duration
7172025-02-07T20:20:34.695Z
7182025-02-07T20:20:34.695Zreal 13:21.742042491
7192025-02-07T20:20:34.695Zuser 18:38.995520054
7202025-02-07T20:20:34.695Zsys 25.697002008
7212025-02-07T20:20:34.695Ztrap 0.546298516
7222025-02-07T20:20:34.695Ztflt 0.044062041
7232025-02-07T20:20:34.695Zdflt 0.052314431
7242025-02-07T20:20:34.695Zkflt 0.001012565
7252025-02-07T20:20:34.695Zlock 25:26:34.960641448
7262025-02-07T20:20:34.695Zslp 1:14:08.617205018
7272025-02-07T20:20:34.695Zlat 2:03.831146036
7282025-02-07T20:20:34.695Zstop 0.249174650
7292025-02-07T20:20:34.695Z+ echo 'test-up-2region-encrypted ends'
7302025-02-07T20:20:34.695Ztest-up-2region-encrypted ends
7312025-02-07T20:20:39.700Zprocess exited: duration 810158 ms, exit code 0
7322025-02-07T20:20:39.700Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
7332025-02-07T20:21:39.727Zexec warning: : stdout descriptor may be held open by a background process; giving up!
7342025-02-07T20:21:39.727Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
7352025-02-07T20:21:39.738Zfound 17 output files
7362025-02-07T20:21:39.738Zuploading: /tmp/test_up-build/dsc-out.txt (48897 bytes)
7372025-02-07T20:21:40.749Zuploaded: /tmp/test_up-build/dsc-out.txt
7382025-02-07T20:21:40.749Zuploading: /tmp/test_up-build/test_up_out.txt (8432802 bytes)
7392025-02-07T20:21:41.881Zuploaded: /tmp/test_up-build/test_up_out.txt
7402025-02-07T20:21:41.882Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (147258 bytes)
7412025-02-07T20:21:42.899Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
7422025-02-07T20:21:42.899Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (144653 bytes)
7432025-02-07T20:21:43.914Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
7442025-02-07T20:21:43.914Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (251014 bytes)
7452025-02-07T20:21:44.934Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
7462025-02-07T20:21:44.934Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (139426 bytes)
7472025-02-07T20:21:45.950Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
7482025-02-07T20:21:45.950Zuploading: /tmp/test_up-build/dsc/downstairs-8850.txt (2500 bytes)
7492025-02-07T20:21:46.956Zuploaded: /tmp/test_up-build/dsc/downstairs-8850.txt
7502025-02-07T20:21:46.957Zuploading: /tmp/test_up-build/dsc/downstairs-8860.txt (33788 bytes)
7512025-02-07T20:21:47.966Zuploaded: /tmp/test_up-build/dsc/downstairs-8860.txt
7522025-02-07T20:21:47.966Zuploading: /tmp/test_up-build/dsc/downstairs-8870.txt (735710 bytes)
7532025-02-07T20:21:48.999Zuploaded: /tmp/test_up-build/dsc/downstairs-8870.txt
7542025-02-07T20:21:48.999Zuploading: /tmp/debug/df.txt (1270 bytes)
7552025-02-07T20:21:50.007Zuploaded: /tmp/debug/df.txt
7562025-02-07T20:21:50.007Zuploading: /tmp/debug/dtrace.txt (966860 bytes)
7572025-02-07T20:21:51.045Zuploaded: /tmp/debug/dtrace.txt
7582025-02-07T20:21:51.045Zuploading: /tmp/debug/iostat.txt (261019 bytes)
7592025-02-07T20:21:51.062Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 261019 -> 264631
7602025-02-07T20:21:52.070Zuploaded: /tmp/debug/iostat.txt
7612025-02-07T20:21:52.070Zuploading: /tmp/debug/mpstat.txt (632922 bytes)
7622025-02-07T20:21:52.101Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 632922 -> 642412
7632025-02-07T20:21:53.111Zuploaded: /tmp/debug/mpstat.txt
7642025-02-07T20:21:53.111Zuploading: /tmp/debug/paging.txt (114318 bytes)
7652025-02-07T20:21:53.123Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 114318 -> 116258
7662025-02-07T20:21:54.129Zuploaded: /tmp/debug/paging.txt
7672025-02-07T20:21:54.129Zuploading: /tmp/debug/prstat.txt (1231609 bytes)
7682025-02-07T20:21:54.168Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 1231609 -> 1249676
7692025-02-07T20:21:55.174Zuploaded: /tmp/debug/prstat.txt
7702025-02-07T20:21:55.174Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
7712025-02-07T20:21:56.182Zuploaded: /tmp/debug/psrinfo.txt
7722025-02-07T20:21:56.182Zuploading: /tmp/debug/upstairs-info.txt (177210 bytes)
7732025-02-07T20:21:57.199Zuploaded: /tmp/debug/upstairs-info.txt