01JMZ5521JD0M5YQC5M0MJHB4T: test-up-2region-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01JMZ55M67V5DG10Z329MBT0Y1

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12025-02-25T20:32:15.824Zjob dependencies complete; ready to run (waiting for 2 h 16 m 6 s)
22025-02-25T20:47:24.388Zjob assigned to worker 01JMZDRWKGFJ5TY3CX1T05F791 [factory aws, i-0964dc6696a98e1f3] (queued for 15 m 8 s)
32025-02-25T20:47:32.100Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42025-02-25T20:47:33.286Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52025-02-25T20:47:33.286Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62025-02-25T20:47:34.339Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72025-02-25T20:47:34.339Zdownloading input: /input/build/work/bins/crutest.gz
82025-02-25T20:47:35.479Zdownloaded input: /input/build/work/bins/crutest.gz
92025-02-25T20:47:35.479Zdownloading input: /input/build/work/bins/dsc.gz
102025-02-25T20:47:36.151Zdownloaded input: /input/build/work/bins/dsc.gz
112025-02-25T20:47:36.154Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122025-02-25T20:47:36.168Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132025-02-25T20:47:36.171Zdownloading input: /input/build/work/scripts/test_ds.sh
142025-02-25T20:47:36.187Zdownloaded input: /input/build/work/scripts/test_ds.sh
152025-02-25T20:47:36.187Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162025-02-25T20:47:36.211Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172025-02-25T20:47:36.211Zdownloading input: /input/build/work/scripts/test_repair.sh
182025-02-25T20:47:36.246Zdownloaded input: /input/build/work/scripts/test_repair.sh
192025-02-25T20:47:36.246Zdownloading input: /input/build/work/scripts/test_replay.sh
202025-02-25T20:47:36.289Zdownloaded input: /input/build/work/scripts/test_replay.sh
212025-02-25T20:47:36.289Zdownloading input: /input/build/work/scripts/test_up.sh
222025-02-25T20:47:36.341Zdownloaded input: /input/build/work/scripts/test_up.sh
232025-02-25T20:47:36.341Zdownloading input: /input/build/work/scripts/upstairs_info.d
242025-02-25T20:47:36.371Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252025-02-25T20:47:36.373Zdownloading input: /input/build/tmp/cargo-test-out.log
262025-02-25T20:47:37.357Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272025-02-25T20:47:37.357Zstarting task 0: "setup"
282025-02-25T20:47:37.360Z++ uname -s
292025-02-25T20:47:37.360Z+ kern=SunOS
302025-02-25T20:47:37.360Z+ build_user=build
312025-02-25T20:47:37.360Z+ build_uid=12345
322025-02-25T20:47:37.360Z+ work_dir=/work
332025-02-25T20:47:37.360Z+ input_dir=/input
342025-02-25T20:47:37.362Z+ [[ 0 == 12345 ]]
352025-02-25T20:47:37.362Z+ case "$kern" in
362025-02-25T20:47:37.362Z+ groupadd -g 12345 build
372025-02-25T20:47:37.365Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382025-02-25T20:47:39.371Z+ zfs create -o mountpoint=/work rpool/work
392025-02-25T20:47:39.558Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402025-02-25T20:47:39.560Z+ home_fs=zfs
412025-02-25T20:47:39.560Z+ [[ zfs == autofs ]]
422025-02-25T20:47:39.561Z+ mkdir -p /home/build
432025-02-25T20:47:39.563Z+ chown build:build /home/build /work
442025-02-25T20:47:41.565Z+ chmod 0700 /home/build /work
452025-02-25T20:47:41.567Zprocess exited: duration 4214 ms, exit code 0
 
462025-02-25T20:47:41.572Zstarting task 1: "authentication"
472025-02-25T20:47:41.590Zprocess exited: duration 17 ms, exit code 0
 
482025-02-25T20:47:41.597Zstarting task 2: "build"
492025-02-25T20:47:41.602Z+ banner cores
502025-02-25T20:47:41.605Z
512025-02-25T20:47:41.605Z #### #### ##### ###### ####
522025-02-25T20:47:41.605Z # # # # # # # #
532025-02-25T20:47:41.605Z # # # # # ##### ####
542025-02-25T20:47:41.605Z # # # ##### # #
552025-02-25T20:47:41.605Z # # # # # # # # #
562025-02-25T20:47:41.605Z #### #### # # ###### ####
572025-02-25T20:47:41.605Z
582025-02-25T20:47:41.609Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
592025-02-25T20:47:41.613Z+ banner unpack
602025-02-25T20:47:41.616Z
612025-02-25T20:47:41.616Z # # # # ##### ## #### # #
622025-02-25T20:47:41.616Z # # ## # # # # # # # # #
632025-02-25T20:47:41.616Z # # # # # # # # # # ####
642025-02-25T20:47:41.616Z # # # # # ##### ###### # # #
652025-02-25T20:47:41.616Z # # # ## # # # # # # #
662025-02-25T20:47:41.616Z #### # # # # # #### # #
672025-02-25T20:47:41.616Z
682025-02-25T20:47:41.616Z+ mkdir -p /var/tmp/bins
692025-02-25T20:47:41.619Z+ for t in "$input/bins/"*.gz
702025-02-25T20:47:41.619Z++ basename /input/build/work/bins/crucible-downstairs.gz
712025-02-25T20:47:41.622Z+ b=crucible-downstairs.gz
722025-02-25T20:47:41.622Z+ b=crucible-downstairs
732025-02-25T20:47:41.622Z+ gunzip
742025-02-25T20:47:44.046Z+ chmod +x /var/tmp/bins/crucible-downstairs
752025-02-25T20:47:44.049Z+ for t in "$input/bins/"*.gz
762025-02-25T20:47:44.052Z++ basename /input/build/work/bins/crucible-hammer.gz
772025-02-25T20:47:44.052Z+ b=crucible-hammer.gz
782025-02-25T20:47:44.052Z+ b=crucible-hammer
792025-02-25T20:47:44.052Z+ gunzip
802025-02-25T20:47:46.216Z+ chmod +x /var/tmp/bins/crucible-hammer
812025-02-25T20:47:46.219Z+ for t in "$input/bins/"*.gz
822025-02-25T20:47:46.222Z++ basename /input/build/work/bins/crutest.gz
832025-02-25T20:47:46.222Z+ b=crutest.gz
842025-02-25T20:47:46.222Z+ b=crutest
852025-02-25T20:47:46.222Z+ gunzip
862025-02-25T20:47:48.660Z+ chmod +x /var/tmp/bins/crutest
872025-02-25T20:47:48.663Z+ for t in "$input/bins/"*.gz
882025-02-25T20:47:48.665Z++ basename /input/build/work/bins/dsc.gz
892025-02-25T20:47:48.665Z+ b=dsc.gz
902025-02-25T20:47:48.665Z+ b=dsc
912025-02-25T20:47:48.665Z+ gunzip
922025-02-25T20:47:50.013Z+ chmod +x /var/tmp/bins/dsc
932025-02-25T20:47:50.016Z+ export BINDIR=/var/tmp/bins
942025-02-25T20:47:50.016Z+ BINDIR=/var/tmp/bins
952025-02-25T20:47:50.016Z+ jobpid=1147
962025-02-25T20:47:50.019Z+ echo 'Setup debug logging'
972025-02-25T20:47:50.019ZSetup debug logging
982025-02-25T20:47:50.019Z+ mkdir /tmp/debug
992025-02-25T20:47:50.019Z+ sleep 3600
1002025-02-25T20:47:50.019Z+ psrinfo -v
1012025-02-25T20:47:50.022Z+ df -h
1022025-02-25T20:47:50.029Z+ prstat -d d -mLc 1
1032025-02-25T20:47:50.032Z+ iostat -T d -xn 1
1042025-02-25T20:47:50.032Z+ mpstat -T d 1
1052025-02-25T20:47:50.032Z+ vmstat -T d -p 1
1062025-02-25T20:47:50.035Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1072025-02-25T20:47:50.035Z+ banner test_up_2r_encrypted
1082025-02-25T20:47:50.035Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1092025-02-25T20:47:50.035Z #####
1102025-02-25T20:47:50.035Z ##### ###### #### ##### # # ##### # # #####
1112025-02-25T20:47:50.035Z # # # # # # # # # # #
1122025-02-25T20:47:50.035Z # ##### #### # # # # # ##### # #
1132025-02-25T20:47:50.035Z # # # # # # ##### # #####
1142025-02-25T20:47:50.035Z # # # # # # # # # # #
1152025-02-25T20:47:50.035Z # ###### #### # ####### #### # ####### ####### # #
1162025-02-25T20:47:50.035Z
1172025-02-25T20:47:50.040Z+ ptime -m bash /input/build/work/scripts/test_up.sh -r 2 -N encrypted
1182025-02-25T20:47:50.047Z/input/build/work
1192025-02-25T20:47:50.058ZUsing 2 region sets
1202025-02-25T20:47:50.058ZTurn off color for downstairs dump
1212025-02-25T20:47:50.073ZUpstairs using key: +VP2u+Ygayg/pNgglBz77ngsROe6iplL/AVzI3y1tqQ=
1222025-02-25T20:47:50.075Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1232025-02-25T20:47:50.076ZCreating 6 downstairs regions
1242025-02-25T20:47:50.432ZStarting 6 downstairs
1252025-02-25T20:47:50.435Zdsc started at PID: 1195
1262025-02-25T20:47:55.773ZDisable automatic restart on all downstairs
1272025-02-25T20:47:55.826Z
1282025-02-25T20:47:55.826ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1292025-02-25T20:47:55.830ZRunning test: span
1302025-02-25T20:48:00.008ZCompleted test: span
1312025-02-25T20:48:05.016ZRunning test: big
1322025-02-25T20:48:09.598ZCompleted test: big
1332025-02-25T20:48:14.606ZRunning test: dep
1342025-02-25T20:48:48.255ZCompleted test: dep
1352025-02-25T20:48:53.263ZRunning test: balloon
1362025-02-25T20:53:07.115ZCompleted test: balloon
1372025-02-25T20:53:12.123ZRunning test: deactivate
1382025-02-25T20:54:06.771ZCompleted test: deactivate
1392025-02-25T20:54:11.778ZRunning hammer
1402025-02-25T20:54:24.388ZRun repair tests
1412025-02-25T20:54:24.392Z/var/tmp/bins/crutest fill -g 31 -q --key +VP2u+Ygayg/pNgglBz77ngsROe6iplL/AVzI3y1tqQ= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1422025-02-25T20:54:24.451Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.725109942Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"task":"crutest"}
1432025-02-25T20:54:24.454Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.72567944Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"task":"crutest"}
1442025-02-25T20:54:24.454Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.727317246Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"task":"crutest"}
1452025-02-25T20:54:24.454Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.727535602Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238}
1462025-02-25T20:54:24.454Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cdb6c7ae221cade941351dba5e763db1f87ef907\",\n git_commit_timestamp: \"2025-02-25T18:15:07.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.727588982Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238}
1472025-02-25T20:54:24.454Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.727616156Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238}
1482025-02-25T20:54:24.454Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.727634843Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238}
1492025-02-25T20:54:24.456Z{"msg":"Crucible 49b2f4c2-80a4-4106-bf52-8e3eec689493 has session id: b7a9c75d-356a-4b3f-a7dd-acb249d14a94","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.72830498Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1502025-02-25T20:54:24.456Z{"msg":"Upstairs opts: Upstairs UUID: 49b2f4c2-80a4-4106-bf52-8e3eec689493, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.728384724Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1512025-02-25T20:54:24.457Z{"msg":"Crucible stats registered with UUID: 49b2f4c2-80a4-4106-bf52-8e3eec689493","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.728411749Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1522025-02-25T20:54:24.457Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-25T20:54:23.728425666Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1532025-02-25T20:54:24.457Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.730308493Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"task":"crutest"}
1542025-02-25T20:54:24.457Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.730395966Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238}
1552025-02-25T20:54:24.457Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cdb6c7ae221cade941351dba5e763db1f87ef907\",\n git_commit_timestamp: \"2025-02-25T18:15:07.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.730430219Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238}
1562025-02-25T20:54:24.457Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.730450035Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238}
1572025-02-25T20:54:24.457Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.730464323Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238}
1582025-02-25T20:54:24.460Z{"msg":"Crucible 49b2f4c2-80a4-4106-bf52-8e3eec689493 has session id: 524a5f51-af77-4bfc-a187-96a802304cd7","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.730906055Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
1592025-02-25T20:54:24.460Z{"msg":"Upstairs opts: Upstairs UUID: 49b2f4c2-80a4-4106-bf52-8e3eec689493, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.730949036Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
1602025-02-25T20:54:24.460Z{"msg":"Crucible stats registered with UUID: 49b2f4c2-80a4-4106-bf52-8e3eec689493","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.730964373Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
1612025-02-25T20:54:24.460Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-25T20:54:23.730976651Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
1622025-02-25T20:54:24.460Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.73189214Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"task":"crutest"}
1632025-02-25T20:54:24.460Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.731914545Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238}
1642025-02-25T20:54:24.460Z{"msg":"49b2f4c2-80a4-4106-bf52-8e3eec689493 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.732021814Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1652025-02-25T20:54:24.460Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.732047299Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"client":"0","":"downstairs","session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1662025-02-25T20:54:24.460Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.732073704Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"client":"0","":"downstairs","session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1672025-02-25T20:54:24.460Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.73209558Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"client":"1","":"downstairs","session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1682025-02-25T20:54:24.460Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.732110057Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"client":"1","":"downstairs","session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1692025-02-25T20:54:24.460Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.732130333Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"client":"2","":"downstairs","session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1702025-02-25T20:54:24.460Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.732155498Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"client":"2","":"downstairs","session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1712025-02-25T20:54:24.460Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.732199129Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1722025-02-25T20:54:24.460Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.732239731Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1732025-02-25T20:54:24.460Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.732266136Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1742025-02-25T20:54:24.460Z{"msg":"ds_connection connected from Ok(127.0.0.1:55022)","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.732313037Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1752025-02-25T20:54:24.460Z{"msg":"ds_connection connected from Ok(127.0.0.1:55019)","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.732360447Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1762025-02-25T20:54:24.460Z{"msg":"ds_connection connected from Ok(127.0.0.1:55892)","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.73244957Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1772025-02-25T20:54:24.460Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.732930614Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"client":"1","":"downstairs","session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1782025-02-25T20:54:24.460Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.732973036Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"client":"0","":"downstairs","session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1792025-02-25T20:54:24.460Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.733010239Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"client":"2","":"downstairs","session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1802025-02-25T20:54:24.464Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.733600292Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"client":"1","":"downstairs","session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1812025-02-25T20:54:24.464Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.733640104Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"client":"2","":"downstairs","session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1822025-02-25T20:54:24.464Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.733678556Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"client":"0","":"downstairs","session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1832025-02-25T20:54:24.464Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.734038625Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1842025-02-25T20:54:24.464Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.734135066Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1852025-02-25T20:54:24.464Z{"msg":"[0]R flush_numbers: [4006, 4006, 4006, 4006, 4006]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.734180297Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"downstairs","":"downstairs","session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1862025-02-25T20:54:24.464Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.734243004Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"downstairs","":"downstairs","session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1872025-02-25T20:54:24.464Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.734278048Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"downstairs","":"downstairs","session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1882025-02-25T20:54:24.464Z{"msg":"[1]R flush_numbers: [4006, 4006, 4006, 4006, 4006]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.734302823Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"downstairs","":"downstairs","session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1892025-02-25T20:54:24.464Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.734333486Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"downstairs","":"downstairs","session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1902025-02-25T20:54:24.464Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.734399953Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"downstairs","":"downstairs","session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1912025-02-25T20:54:24.464Z{"msg":"[2]R flush_numbers: [4006, 4006, 4006, 4006, 4006]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.734444315Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"downstairs","":"downstairs","session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1922025-02-25T20:54:24.464Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.734511131Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"downstairs","":"downstairs","session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1932025-02-25T20:54:24.464Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.734539416Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"downstairs","":"downstairs","session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1942025-02-25T20:54:24.464Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.734571919Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"downstairs","":"downstairs","session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1952025-02-25T20:54:24.464Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.734597534Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"downstairs","":"downstairs","session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1962025-02-25T20:54:24.464Z{"msg":"Next flush: 4007","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.734630118Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"downstairs","":"downstairs","session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1972025-02-25T20:54:24.464Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.734653363Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"downstairs","":"downstairs","session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1982025-02-25T20:54:24.464Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.734756803Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
1992025-02-25T20:54:24.464Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.734833108Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"client":"0","":"downstairs","session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
2002025-02-25T20:54:24.464Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.73487433Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"client":"1","":"downstairs","session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
2012025-02-25T20:54:24.464Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.734900674Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"client":"2","":"downstairs","session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
2022025-02-25T20:54:24.464Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.734965292Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
2032025-02-25T20:54:24.464Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.734994556Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
2042025-02-25T20:54:24.464Z{"msg":"49b2f4c2-80a4-4106-bf52-8e3eec689493 is now active with session: b7a9c75d-356a-4b3f-a7dd-acb249d14a94","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.73502251Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
2052025-02-25T20:54:24.465Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.735066292Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"session_id":"b7a9c75d-356a-4b3f-a7dd-acb249d14a94"}
2062025-02-25T20:54:24.465Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.735081078Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238}
2072025-02-25T20:54:24.465Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.735093306Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238}
2082025-02-25T20:54:24.465Z{"msg":"49b2f4c2-80a4-4106-bf52-8e3eec689493 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.735105624Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2092025-02-25T20:54:24.465Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.735117781Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"client":"0","":"downstairs","session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2102025-02-25T20:54:24.465Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.735157623Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"client":"0","":"downstairs","session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2112025-02-25T20:54:24.465Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.735194616Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"client":"1","":"downstairs","session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2122025-02-25T20:54:24.465Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.73522449Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"client":"1","":"downstairs","session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2132025-02-25T20:54:24.465Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.735264982Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"client":"2","":"downstairs","session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2142025-02-25T20:54:24.465Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.735295086Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"client":"2","":"downstairs","session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2152025-02-25T20:54:24.465Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.735320441Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2162025-02-25T20:54:24.465Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.735341797Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2172025-02-25T20:54:24.465Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.73537574Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2182025-02-25T20:54:24.465Z{"msg":"ds_connection connected from Ok(127.0.0.1:50603)","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.735395426Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2192025-02-25T20:54:24.465Z{"msg":"ds_connection connected from Ok(127.0.0.1:42645)","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.735410523Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2202025-02-25T20:54:24.465Z{"msg":"ds_connection connected from Ok(127.0.0.1:54652)","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.7354265Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2212025-02-25T20:54:24.465Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.735449306Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"client":"1","":"downstairs","session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2222025-02-25T20:54:24.465Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.73547798Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"client":"2","":"downstairs","session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2232025-02-25T20:54:24.465Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.735502665Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"client":"0","":"downstairs","session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2242025-02-25T20:54:24.465Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.735640918Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"client":"1","":"downstairs","session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2252025-02-25T20:54:24.465Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.735768802Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"client":"0","":"downstairs","session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2262025-02-25T20:54:24.465Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.735806485Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"client":"2","":"downstairs","session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2272025-02-25T20:54:24.465Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.736003566Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2282025-02-25T20:54:24.465Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.736151697Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2292025-02-25T20:54:24.465Z{"msg":"[0]R flush_numbers: [5319, 5320, 5322, 5323, 5323]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.73618204Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"downstairs","":"downstairs","session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2302025-02-25T20:54:24.465Z{"msg":"[0]R generation: [13, 13, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.736196758Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"downstairs","":"downstairs","session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2312025-02-25T20:54:24.465Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.736217344Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"downstairs","":"downstairs","session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2322025-02-25T20:54:24.465Z{"msg":"[1]R flush_numbers: [5319, 5320, 5322, 5323, 5323]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.73623721Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"downstairs","":"downstairs","session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2332025-02-25T20:54:24.465Z{"msg":"[1]R generation: [13, 13, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.736250747Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"downstairs","":"downstairs","session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2342025-02-25T20:54:24.465ZWait for a query_work_queue command to finish before sending IO
2352025-02-25T20:54:24.465Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.736304626Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"downstairs","":"downstairs","session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2362025-02-25T20:54:24.465Z{"msg":"[2]R flush_numbers: [5319, 5320, 5322, 5323, 5323]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.736320543Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"downstairs","":"downstairs","session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2372025-02-25T20:54:24.465Z{"msg":"[2]R generation: [13, 13, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.736340219Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"downstairs","":"downstairs","session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2382025-02-25T20:54:24.465Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.736353067Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"downstairs","":"downstairs","session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2392025-02-25T20:54:24.465Z{"msg":"Max found gen is 15","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.736365484Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"downstairs","":"downstairs","session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2402025-02-25T20:54:24.465Z{"msg":"Generation requested: 31 >= found:15","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.736378622Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"downstairs","":"downstairs","session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2412025-02-25T20:54:24.466Z{"msg":"Next flush: 5324","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.736391069Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"downstairs","":"downstairs","session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2422025-02-25T20:54:24.466Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.736404077Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"":"downstairs","":"downstairs","session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2432025-02-25T20:54:24.466Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.736416564Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2442025-02-25T20:54:24.466Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.73643556Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"client":"0","":"downstairs","session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2452025-02-25T20:54:24.466Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.736448748Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"client":"1","":"downstairs","session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2462025-02-25T20:54:24.466Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.736461425Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"client":"2","":"downstairs","session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2472025-02-25T20:54:24.466Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.736474343Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2482025-02-25T20:54:24.466Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.73648657Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2492025-02-25T20:54:24.466Z{"msg":"49b2f4c2-80a4-4106-bf52-8e3eec689493 is now active with session: 524a5f51-af77-4bfc-a187-96a802304cd7","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.736498978Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2502025-02-25T20:54:24.466Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.736511035Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238,"session_id":"524a5f51-af77-4bfc-a187-96a802304cd7"}
2512025-02-25T20:54:24.466Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:23.736522813Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238}
2522025-02-25T20:54:24.466ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
2532025-02-25T20:54:24.466ZFill test
2542025-02-25T20:54:24.505ZRead and Verify all blocks (0..100 range:false)
2552025-02-25T20:54:24.583ZWrote out file "/var/tmp/test_up-build/verify_file"
2562025-02-25T20:54:24.583ZCLIENT: Tests done. All submitted work has been ACK'd
2572025-02-25T20:54:24.586Z----------------------------------------------------------------
2582025-02-25T20:54:24.586Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2592025-02-25T20:54:24.586ZStates: Active Active Active
2602025-02-25T20:54:24.586ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2612025-02-25T20:54:24.586Z 1002 Acked Read 50 Done Done Done false
2622025-02-25T20:54:24.586Z STATES DS:0 DS:1 DS:2 TOTAL
2632025-02-25T20:54:24.586Z Sent 0 0 0 0
2642025-02-25T20:54:24.586Z Done 1 1 1 3
2652025-02-25T20:54:24.586Z Skipped 0 0 0 0
2662025-02-25T20:54:24.586Z Error 0 0 0 0
2672025-02-25T20:54:24.586ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2682025-02-25T20:54:24.586ZDownstairs last five completed: 1001 1000
2692025-02-25T20:54:24.586ZUpstairs last five completed: 1002 1001 1000
2702025-02-25T20:54:24.586Z----------------------------------------------------------------
2712025-02-25T20:54:24.586Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2722025-02-25T20:54:24.586ZStates: Active Active Active
2732025-02-25T20:54:24.586ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2742025-02-25T20:54:24.586Z 1002 Acked Read 50 Done Done Done false
2752025-02-25T20:54:24.586Z STATES DS:0 DS:1 DS:2 TOTAL
2762025-02-25T20:54:24.586Z Sent 0 0 0 0
2772025-02-25T20:54:24.586Z Done 1 1 1 3
2782025-02-25T20:54:24.586Z Skipped 0 0 0 0
2792025-02-25T20:54:24.586Z Error 0 0 0 0
2802025-02-25T20:54:24.586ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2812025-02-25T20:54:24.586ZDownstairs last five completed: 1001 1000
2822025-02-25T20:54:24.586ZUpstairs last five completed: 1002 1001 1000
2832025-02-25T20:54:24.586ZCLIENT: Up:0 ds:2 act:6
2842025-02-25T20:54:28.586Z----------------------------------------------------------------
2852025-02-25T20:54:28.586Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2862025-02-25T20:54:28.586ZDownstairs last five completed: 1003 1002 1001 1000
2872025-02-25T20:54:28.586ZUpstairs last five completed: 1003 1002 1001 1000
2882025-02-25T20:54:28.589Z----------------------------------------------------------------
2892025-02-25T20:54:28.589Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2902025-02-25T20:54:28.589ZDownstairs last five completed: 1003 1002 1001 1000
2912025-02-25T20:54:28.589ZUpstairs last five completed: 1003 1002 1001 1000
2922025-02-25T20:54:28.589ZCLIENT: Up:0 ds:0 act:6
2932025-02-25T20:54:28.589ZCLIENT: All crucible jobs finished, exiting program
2942025-02-25T20:54:28.589Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-25T20:54:27.859658501Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1238}
2952025-02-25T20:54:28.597ZRepair setup passed
2962025-02-25T20:54:28.601ZCopy the region for /var/tmp/test_up-build/8830
2972025-02-25T20:54:28.603Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2982025-02-25T20:54:28.606Z/var/tmp/bins/crutest fill -g 32 -q --key +VP2u+Ygayg/pNgglBz77ngsROe6iplL/AVzI3y1tqQ= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2992025-02-25T20:54:28.662Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.934897038Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"task":"crutest"}
3002025-02-25T20:54:28.665Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.935431772Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"task":"crutest"}
3012025-02-25T20:54:28.665Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.937164559Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"task":"crutest"}
3022025-02-25T20:54:28.665Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.937338884Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244}
3032025-02-25T20:54:28.665Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cdb6c7ae221cade941351dba5e763db1f87ef907\",\n git_commit_timestamp: \"2025-02-25T18:15:07.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.937415999Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244}
3042025-02-25T20:54:28.665Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.93745961Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244}
3052025-02-25T20:54:28.665Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.937485875Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244}
3062025-02-25T20:54:28.667Z{"msg":"Crucible 1ed6182b-5e9c-456e-8882-06d7207174b1 has session id: b7c69438-4231-4502-a8bd-ac0fac0a4b8a","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.938092985Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3072025-02-25T20:54:28.667Z{"msg":"Upstairs opts: Upstairs UUID: 1ed6182b-5e9c-456e-8882-06d7207174b1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.938150463Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3082025-02-25T20:54:28.667Z{"msg":"Crucible stats registered with UUID: 1ed6182b-5e9c-456e-8882-06d7207174b1","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.938175138Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3092025-02-25T20:54:28.668Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-25T20:54:27.938196294Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3102025-02-25T20:54:28.668Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.940145239Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"task":"crutest"}
3112025-02-25T20:54:28.668Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.940233941Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244}
3122025-02-25T20:54:28.668Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cdb6c7ae221cade941351dba5e763db1f87ef907\",\n git_commit_timestamp: \"2025-02-25T18:15:07.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.940276682Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244}
3132025-02-25T20:54:28.668Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.940297238Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244}
3142025-02-25T20:54:28.668Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.940311826Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244}
3152025-02-25T20:54:28.668Z{"msg":"Crucible 1ed6182b-5e9c-456e-8882-06d7207174b1 has session id: 9c1cd680-ecf2-4a98-819f-5c0e3201d9c6","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.940650939Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3162025-02-25T20:54:28.668Z{"msg":"Upstairs opts: Upstairs UUID: 1ed6182b-5e9c-456e-8882-06d7207174b1, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.940717495Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3172025-02-25T20:54:28.670Z{"msg":"Crucible stats registered with UUID: 1ed6182b-5e9c-456e-8882-06d7207174b1","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.940767005Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3182025-02-25T20:54:28.671Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-25T20:54:27.940806688Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3192025-02-25T20:54:28.671Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.941364227Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"task":"crutest"}
3202025-02-25T20:54:28.671Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.941407029Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244}
3212025-02-25T20:54:28.671Z{"msg":"1ed6182b-5e9c-456e-8882-06d7207174b1 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.941477655Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3222025-02-25T20:54:28.671Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.941506509Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"client":"0","":"downstairs","session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3232025-02-25T20:54:28.671Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.94155152Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"client":"0","":"downstairs","session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3242025-02-25T20:54:28.671Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.941640472Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"client":"1","":"downstairs","session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3252025-02-25T20:54:28.671Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.941676425Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"client":"1","":"downstairs","session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3262025-02-25T20:54:28.671Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.941699391Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"client":"2","":"downstairs","session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3272025-02-25T20:54:28.671Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.941761349Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"client":"2","":"downstairs","session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3282025-02-25T20:54:28.671Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.941995232Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3292025-02-25T20:54:28.671Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.942051041Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3302025-02-25T20:54:28.671Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.942085904Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3312025-02-25T20:54:28.671Z{"msg":"ds_connection connected from Ok(127.0.0.1:43483)","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.94210982Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3322025-02-25T20:54:28.671Z{"msg":"ds_connection connected from Ok(127.0.0.1:38576)","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.942134675Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3332025-02-25T20:54:28.671Z{"msg":"ds_connection connected from Ok(127.0.0.1:50095)","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.942168438Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3342025-02-25T20:54:28.671Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.942575917Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"client":"0","":"downstairs","session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3352025-02-25T20:54:28.671Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.942615559Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"client":"2","":"downstairs","session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3362025-02-25T20:54:28.671Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.942642164Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"client":"1","":"downstairs","session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3372025-02-25T20:54:28.671Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.943176388Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"client":"0","":"downstairs","session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3382025-02-25T20:54:28.671Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.943292825Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"client":"1","":"downstairs","session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3392025-02-25T20:54:28.671Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.943331258Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"client":"2","":"downstairs","session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3402025-02-25T20:54:28.671Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.943569721Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3412025-02-25T20:54:28.671Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.943648985Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3422025-02-25T20:54:28.675Z{"msg":"[0]R flush_numbers: [4007, 4007, 4007, 4007, 4007]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.943761593Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"downstairs","":"downstairs","session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3432025-02-25T20:54:28.675Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.943803554Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"downstairs","":"downstairs","session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3442025-02-25T20:54:28.675Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.943829909Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"downstairs","":"downstairs","session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3452025-02-25T20:54:28.675Z{"msg":"[1]R flush_numbers: [4007, 4007, 4007, 4007, 4007]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.943866252Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"downstairs","":"downstairs","session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3462025-02-25T20:54:28.675Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.943883779Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"downstairs","":"downstairs","session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3472025-02-25T20:54:28.675Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.943900535Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"downstairs","":"downstairs","session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3482025-02-25T20:54:28.675Z{"msg":"[2]R flush_numbers: [4007, 4007, 4007, 4007, 4007]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.94392509Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"downstairs","":"downstairs","session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3492025-02-25T20:54:28.675Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.943939137Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"downstairs","":"downstairs","session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3502025-02-25T20:54:28.675Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.943964403Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"downstairs","":"downstairs","session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3512025-02-25T20:54:28.676Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.9439782Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"downstairs","":"downstairs","session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3522025-02-25T20:54:28.676Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.94402484Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"downstairs","":"downstairs","session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3532025-02-25T20:54:28.676Z{"msg":"Next flush: 4008","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.944049546Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"downstairs","":"downstairs","session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3542025-02-25T20:54:28.676Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.944071251Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"downstairs","":"downstairs","session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3552025-02-25T20:54:28.676Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.944094127Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3562025-02-25T20:54:28.676Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.944114863Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"client":"0","":"downstairs","session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3572025-02-25T20:54:28.676Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.944137578Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"client":"1","":"downstairs","session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3582025-02-25T20:54:28.676Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.94432962Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"client":"2","":"downstairs","session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3592025-02-25T20:54:28.676Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.944372952Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3602025-02-25T20:54:28.676Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.944398197Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3612025-02-25T20:54:28.676Z{"msg":"1ed6182b-5e9c-456e-8882-06d7207174b1 is now active with session: b7c69438-4231-4502-a8bd-ac0fac0a4b8a","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.944507255Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3622025-02-25T20:54:28.676Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.944551007Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"session_id":"b7c69438-4231-4502-a8bd-ac0fac0a4b8a"}
3632025-02-25T20:54:28.676Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.944579961Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244}
3642025-02-25T20:54:28.676Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.944603506Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244}
3652025-02-25T20:54:28.676Z{"msg":"1ed6182b-5e9c-456e-8882-06d7207174b1 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.944628081Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3662025-02-25T20:54:28.676Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.944654376Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"client":"0","":"downstairs","session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3672025-02-25T20:54:28.676Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.944698317Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"client":"0","":"downstairs","session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3682025-02-25T20:54:28.677Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.944742089Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"client":"1","":"downstairs","session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3692025-02-25T20:54:28.677Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.944772372Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"client":"1","":"downstairs","session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3702025-02-25T20:54:28.677Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.944798058Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"client":"2","":"downstairs","session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3712025-02-25T20:54:28.677Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.94483644Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"client":"2","":"downstairs","session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3722025-02-25T20:54:28.677Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.944866904Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3732025-02-25T20:54:28.677Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.944911615Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3742025-02-25T20:54:28.677Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.9449356Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3752025-02-25T20:54:28.677Z{"msg":"ds_connection connected from Ok(127.0.0.1:51651)","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.944959846Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3762025-02-25T20:54:28.677Z{"msg":"ds_connection connected from Ok(127.0.0.1:51032)","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.944980411Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3772025-02-25T20:54:28.677Z{"msg":"ds_connection connected from Ok(127.0.0.1:63359)","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.945022733Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3782025-02-25T20:54:28.677Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.945067414Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"client":"0","":"downstairs","session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3792025-02-25T20:54:28.677Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.945111665Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"client":"1","":"downstairs","session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3802025-02-25T20:54:28.677Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.945148008Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"client":"2","":"downstairs","session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3812025-02-25T20:54:28.677Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.945164165Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"client":"0","":"downstairs","session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3822025-02-25T20:54:28.677Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.945185491Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"client":"1","":"downstairs","session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3832025-02-25T20:54:28.677Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.945210946Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"client":"2","":"downstairs","session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3842025-02-25T20:54:28.677Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.945362616Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3852025-02-25T20:54:28.677Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.945456797Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3862025-02-25T20:54:28.677Z{"msg":"[0]R flush_numbers: [5324, 5324, 5324, 5324, 5324]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.94554405Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"downstairs","":"downstairs","session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3872025-02-25T20:54:28.677Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.945571254Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"downstairs","":"downstairs","session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3882025-02-25T20:54:28.677Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.945585572Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"downstairs","":"downstairs","session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3892025-02-25T20:54:28.677Z{"msg":"[1]R flush_numbers: [5324, 5324, 5324, 5324, 5324]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.945599429Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"downstairs","":"downstairs","session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3902025-02-25T20:54:28.677Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.945621355Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"downstairs","":"downstairs","session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3912025-02-25T20:54:28.677Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.945634692Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"downstairs","":"downstairs","session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3922025-02-25T20:54:28.678ZWait for a query_work_queue command to finish before sending IO
3932025-02-25T20:54:28.678Z{"msg":"[2]R flush_numbers: [5324, 5324, 5324, 5324, 5324]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.945659747Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"downstairs","":"downstairs","session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3942025-02-25T20:54:28.678Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.945673154Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"downstairs","":"downstairs","session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3952025-02-25T20:54:28.678Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.945692361Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"downstairs","":"downstairs","session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3962025-02-25T20:54:28.678Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.945718025Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"downstairs","":"downstairs","session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3972025-02-25T20:54:28.678Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.945731383Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"downstairs","":"downstairs","session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3982025-02-25T20:54:28.678Z{"msg":"Next flush: 5325","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.945750499Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"downstairs","":"downstairs","session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
3992025-02-25T20:54:28.678Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.945763196Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"":"downstairs","":"downstairs","session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
4002025-02-25T20:54:28.678Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.945775884Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
4012025-02-25T20:54:28.678Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.945799509Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"client":"0","":"downstairs","session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
4022025-02-25T20:54:28.678Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.945812377Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"client":"1","":"downstairs","session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
4032025-02-25T20:54:28.678Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.945824844Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"client":"2","":"downstairs","session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
4042025-02-25T20:54:28.678Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.945837522Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
4052025-02-25T20:54:28.678Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.945856018Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
4062025-02-25T20:54:28.678Z{"msg":"1ed6182b-5e9c-456e-8882-06d7207174b1 is now active with session: 9c1cd680-ecf2-4a98-819f-5c0e3201d9c6","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.945869495Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
4072025-02-25T20:54:28.678Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.945881343Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244,"session_id":"9c1cd680-ecf2-4a98-819f-5c0e3201d9c6"}
4082025-02-25T20:54:28.678Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:27.94589385Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244}
4092025-02-25T20:54:28.678ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
4102025-02-25T20:54:28.678ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4112025-02-25T20:54:28.678ZFill test
4122025-02-25T20:54:28.715ZRead and Verify all blocks (0..100 range:false)
4132025-02-25T20:54:28.793ZWrote out file "/var/tmp/test_up-build/verify_file"
4142025-02-25T20:54:28.793ZCLIENT: Tests done. All submitted work has been ACK'd
4152025-02-25T20:54:28.796Z----------------------------------------------------------------
4162025-02-25T20:54:28.796Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4172025-02-25T20:54:28.796ZStates: Active Active Active
4182025-02-25T20:54:28.796ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4192025-02-25T20:54:28.796Z 1002 Acked Read 50 Done Done Done false
4202025-02-25T20:54:28.796Z STATES DS:0 DS:1 DS:2 TOTAL
4212025-02-25T20:54:28.796Z Sent 0 0 0 0
4222025-02-25T20:54:28.796Z Done 1 1 1 3
4232025-02-25T20:54:28.796Z Skipped 0 0 0 0
4242025-02-25T20:54:28.796Z Error 0 0 0 0
4252025-02-25T20:54:28.796ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4262025-02-25T20:54:28.796ZDownstairs last five completed: 1001 1000
4272025-02-25T20:54:28.796ZUpstairs last five completed: 1002 1001 1000
4282025-02-25T20:54:28.796Z----------------------------------------------------------------
4292025-02-25T20:54:28.796Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4302025-02-25T20:54:28.796ZStates: Active Active Active
4312025-02-25T20:54:28.796ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4322025-02-25T20:54:28.796Z 1002 Acked Read 50 Done Done Done false
4332025-02-25T20:54:28.796Z STATES DS:0 DS:1 DS:2 TOTAL
4342025-02-25T20:54:28.796Z Sent 0 0 0 0
4352025-02-25T20:54:28.796Z Done 1 1 1 3
4362025-02-25T20:54:28.796Z Skipped 0 0 0 0
4372025-02-25T20:54:28.797Z Error 0 0 0 0
4382025-02-25T20:54:28.797ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4392025-02-25T20:54:28.797ZDownstairs last five completed: 1001 1000
4402025-02-25T20:54:28.797ZUpstairs last five completed: 1002 1001 1000
4412025-02-25T20:54:28.797ZCLIENT: Up:0 ds:2 act:6
4422025-02-25T20:54:32.796Z----------------------------------------------------------------
4432025-02-25T20:54:32.796Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4442025-02-25T20:54:32.796ZDownstairs last five completed: 1003 1002 1001 1000
4452025-02-25T20:54:32.796ZUpstairs last five completed: 1003 1002 1001 1000
4462025-02-25T20:54:32.798Z----------------------------------------------------------------
4472025-02-25T20:54:32.798Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4482025-02-25T20:54:32.798ZDownstairs last five completed: 1003 1002 1001 1000
4492025-02-25T20:54:32.798ZUpstairs last five completed: 1003 1002 1001 1000
4502025-02-25T20:54:32.798ZCLIENT: Up:0 ds:0 act:6
4512025-02-25T20:54:32.799ZCLIENT: All crucible jobs finished, exiting program
4522025-02-25T20:54:32.799Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-25T20:54:32.068185807Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1244}
4532025-02-25T20:54:32.803ZRepair part 1 passed
4542025-02-25T20:54:32.805Z
4552025-02-25T20:54:32.805ZKill the current downstairs
4562025-02-25T20:54:32.971ZDownstairs 2 stopped
4572025-02-25T20:54:32.971Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
4582025-02-25T20:54:32.974ZNow put back the original so we have a mismatch
4592025-02-25T20:54:32.974Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
4602025-02-25T20:54:32.976ZRestart downstairs with old directory
4612025-02-25T20:54:33.029Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4622025-02-25T20:54:33.045ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4632025-02-25T20:54:33.048Z 0 000-009 32 32 31 4008 4008 4007 F F F <---
4642025-02-25T20:54:33.048Z 1 010-019 32 32 31 4008 4008 4007 F F F <---
4652025-02-25T20:54:33.048Z 2 020-029 32 32 31 4008 4008 4007 F F F <---
4662025-02-25T20:54:33.048Z 3 030-039 32 32 31 4008 4008 4007 F F F <---
4672025-02-25T20:54:33.048Z 4 040-049 32 32 31 4008 4008 4007 F F F <---
4682025-02-25T20:54:33.048ZMax gen: 32, Max flush: 4008
4692025-02-25T20:54:33.048ZError: Difference in extent metadata found!
4702025-02-25T20:54:33.051Zdump test found error as expected
4712025-02-25T20:54:33.051Z
4722025-02-25T20:54:33.051Z
4732025-02-25T20:54:33.051Z/var/tmp/bins/crutest verify --range -g 33 -q --key +VP2u+Ygayg/pNgglBz77ngsROe6iplL/AVzI3y1tqQ= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
4742025-02-25T20:54:33.105Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.376955844Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"task":"crutest"}
4752025-02-25T20:54:33.108Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.377537389Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"task":"crutest"}
4762025-02-25T20:54:33.108Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.379216946Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"task":"crutest"}
4772025-02-25T20:54:33.108Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.379383003Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252}
4782025-02-25T20:54:33.108Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cdb6c7ae221cade941351dba5e763db1f87ef907\",\n git_commit_timestamp: \"2025-02-25T18:15:07.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.379434643Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252}
4792025-02-25T20:54:33.108Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.379458119Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252}
4802025-02-25T20:54:33.108Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.379473265Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252}
4812025-02-25T20:54:33.111Z{"msg":"Crucible 932799c8-fcc1-4819-91ea-84638a4fed0a has session id: 48a5a63b-62f9-42cf-9285-c6204cf09747","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.380094473Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
4822025-02-25T20:54:33.111Z{"msg":"Upstairs opts: Upstairs UUID: 932799c8-fcc1-4819-91ea-84638a4fed0a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.380144592Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
4832025-02-25T20:54:33.111Z{"msg":"Crucible stats registered with UUID: 932799c8-fcc1-4819-91ea-84638a4fed0a","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.380162719Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
4842025-02-25T20:54:33.111Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-25T20:54:32.380194483Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
4852025-02-25T20:54:33.111Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.382060373Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"task":"crutest"}
4862025-02-25T20:54:33.111Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.382140688Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252}
4872025-02-25T20:54:33.111Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cdb6c7ae221cade941351dba5e763db1f87ef907\",\n git_commit_timestamp: \"2025-02-25T18:15:07.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.38217686Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252}
4882025-02-25T20:54:33.111Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.382190587Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252}
4892025-02-25T20:54:33.111Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.382204995Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252}
4902025-02-25T20:54:33.111Z{"msg":"Crucible 932799c8-fcc1-4819-91ea-84638a4fed0a has session id: 1473b5a3-1ba9-448a-b41f-fbbacd8ffade","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.382596187Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
4912025-02-25T20:54:33.111Z{"msg":"Upstairs opts: Upstairs UUID: 932799c8-fcc1-4819-91ea-84638a4fed0a, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.382650187Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
4922025-02-25T20:54:33.111Z{"msg":"Crucible stats registered with UUID: 932799c8-fcc1-4819-91ea-84638a4fed0a","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.382674992Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
4932025-02-25T20:54:33.111Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-25T20:54:32.382707325Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
4942025-02-25T20:54:33.115Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.383343669Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"task":"crutest"}
4952025-02-25T20:54:33.115Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.383380832Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252}
4962025-02-25T20:54:33.115Z{"msg":"932799c8-fcc1-4819-91ea-84638a4fed0a active request set","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.383504167Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
4972025-02-25T20:54:33.115Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.383534421Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"0","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
4982025-02-25T20:54:33.115Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.383585761Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"0","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
4992025-02-25T20:54:33.115Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.383611246Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"1","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5002025-02-25T20:54:33.115Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.38364399Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"1","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5012025-02-25T20:54:33.115Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.383667295Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"2","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5022025-02-25T20:54:33.115Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.383690631Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"2","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5032025-02-25T20:54:33.115Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.383712476Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5042025-02-25T20:54:33.115Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.383731193Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5052025-02-25T20:54:33.115Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.383751849Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5062025-02-25T20:54:33.115Z{"msg":"ds_connection connected from Ok(127.0.0.1:51169)","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.383783962Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5072025-02-25T20:54:33.115Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-25T20:54:32.383801469Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5082025-02-25T20:54:33.115Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-25T20:54:32.383834062Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5092025-02-25T20:54:33.115Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-25T20:54:32.383929853Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5102025-02-25T20:54:33.115Z{"msg":"ds_connection connected from Ok(127.0.0.1:35296)","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.383970805Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5112025-02-25T20:54:33.115Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.384194001Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5122025-02-25T20:54:33.116Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.384361408Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"0","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5132025-02-25T20:54:33.116Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.384511798Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"1","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5142025-02-25T20:54:33.116Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.384926556Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"0","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5152025-02-25T20:54:33.116Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.385061179Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"1","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5162025-02-25T20:54:33.116Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.385324607Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5172025-02-25T20:54:33.116Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:32.385402592Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5182025-02-25T20:54:43.125Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.385163163Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5192025-02-25T20:54:43.141Z{"msg":"ds_connection connected from Ok(127.0.0.1:46506)","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.385393837Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5202025-02-25T20:54:43.141Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.386636081Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"2","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5212025-02-25T20:54:43.142Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.387172235Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"2","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5222025-02-25T20:54:43.142Z{"msg":"[0]R flush_numbers: [4008, 4008, 4008, 4008, 4008]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.387546451Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"downstairs","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5232025-02-25T20:54:43.142Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.387573445Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"downstairs","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5242025-02-25T20:54:43.142Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.387588542Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"downstairs","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5252025-02-25T20:54:43.142Z{"msg":"[1]R flush_numbers: [4008, 4008, 4008, 4008, 4008]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.38760264Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"downstairs","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5262025-02-25T20:54:43.142Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.387623265Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"downstairs","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5272025-02-25T20:54:43.142Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.387637233Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"downstairs","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5282025-02-25T20:54:43.143Z{"msg":"[2]R flush_numbers: [4007, 4007, 4007, 4007, 4007]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.387657259Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"downstairs","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5292025-02-25T20:54:43.143Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.387670326Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"downstairs","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5302025-02-25T20:54:43.143Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.387690042Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"downstairs","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5312025-02-25T20:54:43.143Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.38770292Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"downstairs","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5322025-02-25T20:54:43.143Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.387722476Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"downstairs","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5332025-02-25T20:54:43.143Z{"msg":"Next flush: 4009","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.387735723Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"downstairs","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5342025-02-25T20:54:43.143Z{"msg":"extent 0 needs reconciliation: ExtentMetadata { gen: 32, flush: 4008, dirty: false } ExtentMetadata { gen: 32, flush: 4008, dirty: false } ExtentMetadata { gen: 31, flush: 4007, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.387757439Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5352025-02-25T20:54:43.143Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 4008, dirty: false }, ExtentMetadata { gen: 32, flush: 4008, dirty: false }, ExtentMetadata { gen: 31, flush: 4007, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.387773766Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5362025-02-25T20:54:43.144Z{"msg":"found dest for source 0 for extent at index 0 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.387794242Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5372025-02-25T20:54:43.144Z{"msg":"extent 1 needs reconciliation: ExtentMetadata { gen: 32, flush: 4008, dirty: false } ExtentMetadata { gen: 32, flush: 4008, dirty: false } ExtentMetadata { gen: 31, flush: 4007, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.387809768Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5382025-02-25T20:54:43.144Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 4008, dirty: false }, ExtentMetadata { gen: 32, flush: 4008, dirty: false }, ExtentMetadata { gen: 31, flush: 4007, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.387831194Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5392025-02-25T20:54:43.144Z{"msg":"found dest for source 0 for extent at index 1 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.387844912Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5402025-02-25T20:54:43.144Z{"msg":"extent 2 needs reconciliation: ExtentMetadata { gen: 32, flush: 4008, dirty: false } ExtentMetadata { gen: 32, flush: 4008, dirty: false } ExtentMetadata { gen: 31, flush: 4007, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.387866717Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5412025-02-25T20:54:43.144Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 4008, dirty: false }, ExtentMetadata { gen: 32, flush: 4008, dirty: false }, ExtentMetadata { gen: 31, flush: 4007, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.387882214Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5422025-02-25T20:54:43.145Z{"msg":"found dest for source 0 for extent at index 2 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.38790242Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5432025-02-25T20:54:43.145Z{"msg":"extent 3 needs reconciliation: ExtentMetadata { gen: 32, flush: 4008, dirty: false } ExtentMetadata { gen: 32, flush: 4008, dirty: false } ExtentMetadata { gen: 31, flush: 4007, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.387919157Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5442025-02-25T20:54:43.145Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 4008, dirty: false }, ExtentMetadata { gen: 32, flush: 4008, dirty: false }, ExtentMetadata { gen: 31, flush: 4007, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.387941353Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5452025-02-25T20:54:43.145Z{"msg":"found dest for source 0 for extent at index 3 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.387975306Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5462025-02-25T20:54:43.145Z{"msg":"extent 4 needs reconciliation: ExtentMetadata { gen: 32, flush: 4008, dirty: false } ExtentMetadata { gen: 32, flush: 4008, dirty: false } ExtentMetadata { gen: 31, flush: 4007, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.387991383Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5472025-02-25T20:54:43.145Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 4008, dirty: false }, ExtentMetadata { gen: 32, flush: 4008, dirty: false }, ExtentMetadata { gen: 31, flush: 4007, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.388006789Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5482025-02-25T20:54:43.146Z{"msg":"found dest for source 0 for extent at index 4 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.388027485Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5492025-02-25T20:54:43.146Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.388042022Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"0","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5502025-02-25T20:54:43.146Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.388063388Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"1","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5512025-02-25T20:54:43.146Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.388076926Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"2","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5522025-02-25T20:54:43.146Z{"msg":"Full repair list: {ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.388100771Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"downstairs","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5532025-02-25T20:54:43.146Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 4009, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 4009, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 4009, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 4009, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 4009, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.388184484Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"downstairs","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5542025-02-25T20:54:43.146Z{"msg":"starting reconciliation 8f495345-6d31-457c-a0d6-f645e8650f68: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.388200791Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"downstairs","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5552025-02-25T20:54:43.146Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.388833326Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"2","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5562025-02-25T20:54:43.146Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.397782724Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"2","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5572025-02-25T20:54:43.147Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.401446759Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"2","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5582025-02-25T20:54:43.158Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.405189228Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"2","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5592025-02-25T20:54:43.158Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.408761071Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"2","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5602025-02-25T20:54:43.158Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.411827194Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"downstairs","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5612025-02-25T20:54:43.158Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.411864007Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"0","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5622025-02-25T20:54:43.159Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.411880134Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"1","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5632025-02-25T20:54:43.159Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.411894551Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"2","":"downstairs","session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5642025-02-25T20:54:43.159Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.411916666Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5652025-02-25T20:54:43.159Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.411929394Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5662025-02-25T20:54:43.159Z{"msg":"932799c8-fcc1-4819-91ea-84638a4fed0a is now active with session: 48a5a63b-62f9-42cf-9285-c6204cf09747","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.41194949Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"session_id":"48a5a63b-62f9-42cf-9285-c6204cf09747"}
5672025-02-25T20:54:43.159Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.411961978Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252}
5682025-02-25T20:54:43.160Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.411979584Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252}
5692025-02-25T20:54:43.160Z{"msg":"932799c8-fcc1-4819-91ea-84638a4fed0a active request set","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.412045031Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
5702025-02-25T20:54:43.160Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.412069306Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"0","":"downstairs","session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
5712025-02-25T20:54:43.160Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.412083773Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"0","":"downstairs","session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
5722025-02-25T20:54:43.160Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.412096751Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"1","":"downstairs","session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
5732025-02-25T20:54:43.160Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.412110558Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"1","":"downstairs","session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
5742025-02-25T20:54:43.160Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.412125915Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"2","":"downstairs","session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
5752025-02-25T20:54:43.161Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.412155199Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"2","":"downstairs","session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
5762025-02-25T20:54:43.161Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.412175795Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
5772025-02-25T20:54:43.161Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.412194811Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
5782025-02-25T20:54:43.161Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.412224955Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
5792025-02-25T20:54:43.161Z{"msg":"ds_connection connected from Ok(127.0.0.1:53694)","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.41225134Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
5802025-02-25T20:54:43.161Z{"msg":"ds_connection connected from Ok(127.0.0.1:44388)","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.412275176Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
5812025-02-25T20:54:43.161Z{"msg":"ds_connection connected from Ok(127.0.0.1:35894)","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.412300131Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
5822025-02-25T20:54:43.161Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.412682525Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"0","":"downstairs","session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
5832025-02-25T20:54:43.162Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.412730725Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"1","":"downstairs","session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
5842025-02-25T20:54:43.162Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.412803851Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"2","":"downstairs","session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
5852025-02-25T20:54:43.162Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.413218839Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"0","":"downstairs","session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
5862025-02-25T20:54:43.162Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.413271668Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"2","":"downstairs","session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
5872025-02-25T20:54:43.162Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.413317639Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"1","":"downstairs","session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
5882025-02-25T20:54:43.162Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.413549193Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
5892025-02-25T20:54:43.162Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.413638335Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
5902025-02-25T20:54:43.162Z{"msg":"[0]R flush_numbers: [5325, 5325, 5325, 5325, 5325]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.41366481Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"downstairs","":"downstairs","session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
5912025-02-25T20:54:43.163Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.413681467Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"downstairs","":"downstairs","session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
5922025-02-25T20:54:43.163Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.413713291Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"downstairs","":"downstairs","session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
5932025-02-25T20:54:43.163Z{"msg":"[1]R flush_numbers: [5325, 5325, 5325, 5325, 5325]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.413731637Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"downstairs","":"downstairs","session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
5942025-02-25T20:54:43.163Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.413755632Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"downstairs","":"downstairs","session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
5952025-02-25T20:54:43.163Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.413788946Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"downstairs","":"downstairs","session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
5962025-02-25T20:54:43.163Z{"msg":"[2]R flush_numbers: [5325, 5325, 5325, 5325, 5325]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.413813341Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO
5972025-02-25T20:54:43.163Z,"session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
5982025-02-25T20:54:43.163Z{"msg":"[2]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.413849714Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"downstairs","":"downstairs","session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
5992025-02-25T20:54:43.163Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.413872149Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"downstairs","":"downstairs","session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
6002025-02-25T20:54:43.164Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.413894155Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"downstairs","":"downstairs","session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
6012025-02-25T20:54:43.164Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.413916031Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"downstairs","":"downstairs","session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
6022025-02-25T20:54:43.164Z{"msg":"Next flush: 5326","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.413938756Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"downstairs","":"downstairs","session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
6032025-02-25T20:54:43.164Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.41397053Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"":"downstairs","":"downstairs","session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
6042025-02-25T20:54:43.164Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.413994395Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
6052025-02-25T20:54:43.164Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.414024649Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"0","":"downstairs","session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
6062025-02-25T20:54:43.164Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.414048225Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"1","":"downstairs","session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
6072025-02-25T20:54:43.165Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.414079028Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"client":"2","":"downstairs","session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
6082025-02-25T20:54:43.165Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.414102004Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
6092025-02-25T20:54:43.165Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.414131658Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
6102025-02-25T20:54:43.165Z{"msg":"932799c8-fcc1-4819-91ea-84638a4fed0a is now active with session: 1473b5a3-1ba9-448a-b41f-fbbacd8ffade","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.414154573Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
6112025-02-25T20:54:43.165Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.414188447Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252,"session_id":"1473b5a3-1ba9-448a-b41f-fbbacd8ffade"}
6122025-02-25T20:54:43.165Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-02-25T20:54:42.414213152Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252}
6132025-02-25T20:54:43.165ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
6142025-02-25T20:54:43.165ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
6152025-02-25T20:54:43.165ZRead and Verify all blocks (0..100 range:true)
6162025-02-25T20:54:43.249ZWrote out file "/var/tmp/test_up-build/verify_file"
6172025-02-25T20:54:43.249ZVerify test completed
6182025-02-25T20:54:43.255ZWrote out file "/var/tmp/test_up-build/verify_file"
6192025-02-25T20:54:43.255ZCLIENT: Tests done. All submitted work has been ACK'd
6202025-02-25T20:54:43.256Z----------------------------------------------------------------
6212025-02-25T20:54:43.256Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6222025-02-25T20:54:43.256ZStates: Active Active Active
6232025-02-25T20:54:43.256ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6242025-02-25T20:54:43.256Z 1000 Acked Read 50 Done Done Done false
6252025-02-25T20:54:43.256Z STATES DS:0 DS:1 DS:2 TOTAL
6262025-02-25T20:54:43.256Z Sent 0 0 0 0
6272025-02-25T20:54:43.256Z Done 1 1 1 3
6282025-02-25T20:54:43.256Z Skipped 0 0 0 0
6292025-02-25T20:54:43.256Z Error 0 0 0 0
6302025-02-25T20:54:43.257ZLast Flush: None None None
6312025-02-25T20:54:43.257ZDownstairs last five completed:
6322025-02-25T20:54:43.257ZUpstairs last five completed: 1000
6332025-02-25T20:54:43.257Z----------------------------------------------------------------
6342025-02-25T20:54:43.257Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6352025-02-25T20:54:43.257ZStates: Active Active Active
6362025-02-25T20:54:43.257ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6372025-02-25T20:54:43.257Z 1000 Acked Read 50 Done Done Done false
6382025-02-25T20:54:43.257Z STATES DS:0 DS:1 DS:2 TOTAL
6392025-02-25T20:54:43.257Z Sent 0 0 0 0
6402025-02-25T20:54:43.257Z Done 1 1 1 3
6412025-02-25T20:54:43.257Z Skipped 0 0 0 0
6422025-02-25T20:54:43.258Z Error 0 0 0 0
6432025-02-25T20:54:43.258ZLast Flush: None None None
6442025-02-25T20:54:43.258ZDownstairs last five completed:
6452025-02-25T20:54:43.258ZUpstairs last five completed: 1000
6462025-02-25T20:54:43.258ZCLIENT: Up:0 ds:2 act:6
6472025-02-25T20:54:47.231Z----------------------------------------------------------------
6482025-02-25T20:54:47.231Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6492025-02-25T20:54:47.231ZDownstairs last five completed: 1001 1000
6502025-02-25T20:54:47.231ZUpstairs last five completed: 1001 1000
6512025-02-25T20:54:47.235Z----------------------------------------------------------------
6522025-02-25T20:54:47.235Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6532025-02-25T20:54:47.235ZDownstairs last five completed: 1001 1000
6542025-02-25T20:54:47.235ZUpstairs last five completed: 1001 1000
6552025-02-25T20:54:47.235ZCLIENT: Up:0 ds:0 act:6
6562025-02-25T20:54:47.235ZCLIENT: All crucible jobs finished, exiting program
6572025-02-25T20:54:47.235Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-25T20:54:46.500690927Z","hostname":"w-01JMZDRWKGFJ5TY3CX1T05F791","pid":1252}
6582025-02-25T20:54:47.239ZRepair part 2 passed
6592025-02-25T20:54:47.241Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
6602025-02-25T20:54:47.257ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6612025-02-25T20:54:47.261Z 0 000-009 32 32 32 4008 4008 4008 F F F
6622025-02-25T20:54:47.261Z 1 010-019 32 32 32 4008 4008 4008 F F F
6632025-02-25T20:54:47.261Z 2 020-029 32 32 32 4008 4008 4008 F F F
6642025-02-25T20:54:47.261Z 3 030-039 32 32 32 4008 4008 4008 F F F
6652025-02-25T20:54:47.261Z 4 040-049 32 32 32 4008 4008 4008 F F F
6662025-02-25T20:54:47.261ZMax gen: 32, Max flush: 4008
6672025-02-25T20:54:47.264Zdump test passed
6682025-02-25T20:54:47.264Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
6692025-02-25T20:54:47.278Z Extent 1
6702025-02-25T20:54:47.280ZGEN 32 32 32
6712025-02-25T20:54:47.281ZFLUSH_ID 4008 4008 4008
6722025-02-25T20:54:47.281ZDIRTY
6732025-02-25T20:54:47.281Z
6742025-02-25T20:54:47.281ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
6752025-02-25T20:54:47.281Z010 A A A A A A
6762025-02-25T20:54:47.281Z011 A A A A A A
6772025-02-25T20:54:47.283Z012 A A A A A A
6782025-02-25T20:54:47.283Z013 A A A A A A
6792025-02-25T20:54:47.285Z014 A A A A A A
6802025-02-25T20:54:47.286Z015 A A A A A A
6812025-02-25T20:54:47.288Z016 A A A A A A
6822025-02-25T20:54:47.288Z017 A A A A A A
6832025-02-25T20:54:47.291Z018 A A A A A A
6842025-02-25T20:54:47.291Z019 A A A A A A
6852025-02-25T20:54:47.294Zdump extent test passed
6862025-02-25T20:54:47.294Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
6872025-02-25T20:54:47.308ZExtent 2 Block in extent 0 Actual block 20
6882025-02-25T20:54:47.309Z
6892025-02-25T20:54:47.311Z DATA SHA256 VER
6902025-02-25T20:54:47.311Z------ ---------------------------------------------------------------- ---
6912025-02-25T20:54:47.311Z 0 c5a9df3dfd22b2dc12de50eb0ec0e696296ec5126e3dd89c55837d673d88701f A
6922025-02-25T20:54:47.314Z 1 c5a9df3dfd22b2dc12de50eb0ec0e696296ec5126e3dd89c55837d673d88701f A
6932025-02-25T20:54:47.314Z 2 c5a9df3dfd22b2dc12de50eb0ec0e696296ec5126e3dd89c55837d673d88701f A
6942025-02-25T20:54:47.314Z
6952025-02-25T20:54:47.314ZNONCES 0 1 2 DIFF
6962025-02-25T20:54:47.314Z------ ------------------------ ------------------------ ------------------------ -----
6972025-02-25T20:54:47.314Z 0 1223d01014db71dbf4c9a7aa 1223d01014db71dbf4c9a7aa 1223d01014db71dbf4c9a7aa
6982025-02-25T20:54:47.314Z
6992025-02-25T20:54:47.314Z TAGS 0 1 2 DIFF
7002025-02-25T20:54:47.314Z------ -------------------------------- -------------------------------- -------------------------------- -----
7012025-02-25T20:54:47.314Z 0 feb350a3f373392936ad5bc6d9f53358 feb350a3f373392936ad5bc6d9f53358 feb350a3f373392936ad5bc6d9f53358
7022025-02-25T20:54:47.314Z
7032025-02-25T20:54:47.314ZHASHES 0 1 2 DIFF
7042025-02-25T20:54:47.314Z------ ---------------- ---------------- ---------------- -----
7052025-02-25T20:54:47.314Z
7062025-02-25T20:54:47.317Zdump block test passed
7072025-02-25T20:54:47.317ZInitial upstairs tests have completed, stopping all downstairs
7082025-02-25T20:54:52.381ZCreating 7 larger downstairs regions
7092025-02-25T20:54:52.569ZStarting 7 downstairs
7102025-02-25T20:54:57.577Zdsc restarted at PID: 1268
7112025-02-25T20:54:57.958ZNow do the replace-reconcile test
7122025-02-25T20:54:58.012ZUsing 8870 for the replacement port
7132025-02-25T20:58:00.890ZCompleted test: replace-reconcile
7142025-02-25T20:58:00.893ZNow do the replace-before-active test
7152025-02-25T21:01:34.407ZCompleted test: replace-before-active
7162025-02-25T21:01:34.407ZAll tests have completed, stopping all downstairs
7172025-02-25T21:01:34.467Z
7182025-02-25T21:01:34.467ZAll Tests have passed
7192025-02-25T21:01:34.472Z13:44 Test duration
7202025-02-25T21:01:34.472Z
7212025-02-25T21:01:34.472Zreal 13:44.241901912
7222025-02-25T21:01:34.472Zuser 18:32.022702102
7232025-02-25T21:01:34.472Zsys 25.014043988
7242025-02-25T21:01:34.472Ztrap 0.535090466
7252025-02-25T21:01:34.472Ztflt 0.041446828
7262025-02-25T21:01:34.472Zdflt 0.034173017
7272025-02-25T21:01:34.472Zkflt 0.001105013
7282025-02-25T21:01:34.472Zlock 25:30:22.625124242
7292025-02-25T21:01:34.472Zslp 1:14:38.342989988
7302025-02-25T21:01:34.473Zlat 2:04.250200573
7312025-02-25T21:01:34.473Zstop 0.227622755
7322025-02-25T21:01:34.473Z+ echo 'test-up-2region-encrypted ends'
7332025-02-25T21:01:34.473Ztest-up-2region-encrypted ends
7342025-02-25T21:01:39.475Zprocess exited: duration 832683 ms, exit code 0
7352025-02-25T21:01:39.475Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
7362025-02-25T21:02:39.530Zexec warning: : stdout descriptor may be held open by a background process; giving up!
7372025-02-25T21:02:39.530Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
7382025-02-25T21:02:39.544Zfound 17 output files
7392025-02-25T21:02:39.544Zuploading: /tmp/test_up-build/dsc-out.txt (52180 bytes)
7402025-02-25T21:02:40.564Zuploaded: /tmp/test_up-build/dsc-out.txt
7412025-02-25T21:02:40.564Zuploading: /tmp/test_up-build/test_up_out.txt (8602820 bytes)
7422025-02-25T21:02:41.726Zuploaded: /tmp/test_up-build/test_up_out.txt
7432025-02-25T21:02:41.729Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (147256 bytes)
7442025-02-25T21:02:42.744Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
7452025-02-25T21:02:42.744Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (144650 bytes)
7462025-02-25T21:02:43.763Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
7472025-02-25T21:02:43.763Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (251041 bytes)
7482025-02-25T21:02:44.778Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
7492025-02-25T21:02:44.781Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (139435 bytes)
7502025-02-25T21:02:45.791Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
7512025-02-25T21:02:45.794Zuploading: /tmp/test_up-build/dsc/downstairs-8850.txt (2503 bytes)
7522025-02-25T21:02:46.801Zuploaded: /tmp/test_up-build/dsc/downstairs-8850.txt
7532025-02-25T21:02:46.806Zuploading: /tmp/test_up-build/dsc/downstairs-8860.txt (33792 bytes)
7542025-02-25T21:02:47.818Zuploaded: /tmp/test_up-build/dsc/downstairs-8860.txt
7552025-02-25T21:02:47.820Zuploading: /tmp/test_up-build/dsc/downstairs-8870.txt (738412 bytes)
7562025-02-25T21:02:48.838Zuploaded: /tmp/test_up-build/dsc/downstairs-8870.txt
7572025-02-25T21:02:48.844Zuploading: /tmp/debug/df.txt (1270 bytes)
7582025-02-25T21:02:49.850Zuploaded: /tmp/debug/df.txt
7592025-02-25T21:02:49.852Zuploading: /tmp/debug/dtrace.txt (962753 bytes)
7602025-02-25T21:02:50.878Zuploaded: /tmp/debug/dtrace.txt
7612025-02-25T21:02:50.880Zuploading: /tmp/debug/iostat.txt (267952 bytes)
7622025-02-25T21:02:50.896Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 267952 -> 271263
7632025-02-25T21:02:51.901Zuploaded: /tmp/debug/iostat.txt
7642025-02-25T21:02:51.901Zuploading: /tmp/debug/mpstat.txt (649713 bytes)
7652025-02-25T21:02:51.927Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 649713 -> 658473
7662025-02-25T21:02:52.933Zuploaded: /tmp/debug/mpstat.txt
7672025-02-25T21:02:52.936Zuploading: /tmp/debug/paging.txt (117302 bytes)
7682025-02-25T21:02:52.944Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 117302 -> 119126
7692025-02-25T21:02:53.956Zuploaded: /tmp/debug/paging.txt
7702025-02-25T21:02:53.956Zuploading: /tmp/debug/prstat.txt (1264188 bytes)
7712025-02-25T21:02:54.050Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 1264188 -> 1283612
7722025-02-25T21:02:55.076Zuploaded: /tmp/debug/prstat.txt
7732025-02-25T21:02:55.078Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
7742025-02-25T21:02:56.085Zuploaded: /tmp/debug/psrinfo.txt
7752025-02-25T21:02:56.085Zuploading: /tmp/debug/upstairs-info.txt (182490 bytes)
7762025-02-25T21:02:57.103Zuploaded: /tmp/debug/upstairs-info.txt