01JHV0SMRYHZDX1B8AS34350AR: test-up-2region-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01JHV0T0A1V4GE524KSCFEE460

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12025-01-17T21:09:38.381Zjob dependencies complete; ready to run (waiting for 13 m 47 s)
22025-01-17T21:10:40.229Zjob assigned to worker 01JHV1KGN2QVJYZ64NNY2VWD61 [factory aws, i-074b65af9a4d6f92a] (queued for 1 m 1 s)
32025-01-17T21:10:48.077Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42025-01-17T21:10:49.254Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52025-01-17T21:10:49.254Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62025-01-17T21:10:50.313Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72025-01-17T21:10:50.313Zdownloading input: /input/build/work/bins/crutest.gz
82025-01-17T21:10:51.486Zdownloaded input: /input/build/work/bins/crutest.gz
92025-01-17T21:10:51.486Zdownloading input: /input/build/work/bins/dsc.gz
102025-01-17T21:10:52.194Zdownloaded input: /input/build/work/bins/dsc.gz
112025-01-17T21:10:52.198Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122025-01-17T21:10:52.215Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132025-01-17T21:10:52.215Zdownloading input: /input/build/work/scripts/test_ds.sh
142025-01-17T21:10:52.234Zdownloaded input: /input/build/work/scripts/test_ds.sh
152025-01-17T21:10:52.234Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162025-01-17T21:10:52.251Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172025-01-17T21:10:52.251Zdownloading input: /input/build/work/scripts/test_repair.sh
182025-01-17T21:10:52.271Zdownloaded input: /input/build/work/scripts/test_repair.sh
192025-01-17T21:10:52.271Zdownloading input: /input/build/work/scripts/test_replay.sh
202025-01-17T21:10:52.289Zdownloaded input: /input/build/work/scripts/test_replay.sh
212025-01-17T21:10:52.289Zdownloading input: /input/build/work/scripts/test_up.sh
222025-01-17T21:10:52.310Zdownloaded input: /input/build/work/scripts/test_up.sh
232025-01-17T21:10:52.310Zdownloading input: /input/build/work/scripts/upstairs_info.d
242025-01-17T21:10:52.331Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252025-01-17T21:10:52.337Zdownloading input: /input/build/tmp/cargo-test-out.log
262025-01-17T21:10:53.044Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272025-01-17T21:10:53.045Zstarting task 0: "setup"
282025-01-17T21:10:53.049Z++ uname -s
292025-01-17T21:10:53.052Z+ kern=SunOS
302025-01-17T21:10:53.052Z+ case "$kern" in
312025-01-17T21:10:53.052Z+ groupadd -g 12345 build
322025-01-17T21:10:53.056Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332025-01-17T21:10:55.062Z+ zfs create -o mountpoint=/work rpool/work
342025-01-17T21:10:55.209Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352025-01-17T21:10:55.213Z+ home_fs=zfs
362025-01-17T21:10:55.213Z+ [[ zfs == autofs ]]
372025-01-17T21:10:55.213Z+ mkdir -p /home/build
382025-01-17T21:10:55.216Z+ chown build:build /home/build /work
392025-01-17T21:10:57.217Z+ chmod 0700 /home/build /work
402025-01-17T21:10:57.220Zprocess exited: duration 4182 ms, exit code 0
 
412025-01-17T21:10:57.227Zstarting task 1: "authentication"
422025-01-17T21:10:57.246Zprocess exited: duration 17 ms, exit code 0
 
432025-01-17T21:10:57.255Zstarting task 2: "build"
442025-01-17T21:10:57.258Z+ banner cores
452025-01-17T21:10:57.265Z
462025-01-17T21:10:57.265Z #### #### ##### ###### ####
472025-01-17T21:10:57.265Z # # # # # # # #
482025-01-17T21:10:57.265Z # # # # # ##### ####
492025-01-17T21:10:57.265Z # # # ##### # #
502025-01-17T21:10:57.265Z # # # # # # # # #
512025-01-17T21:10:57.265Z #### #### # # ###### ####
522025-01-17T21:10:57.265Z
532025-01-17T21:10:57.266Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542025-01-17T21:10:57.270Z+ banner unpack
552025-01-17T21:10:57.273Z
562025-01-17T21:10:57.273Z # # # # ##### ## #### # #
572025-01-17T21:10:57.273Z # # ## # # # # # # # # #
582025-01-17T21:10:57.273Z # # # # # # # # # # ####
592025-01-17T21:10:57.273Z # # # # # ##### ###### # # #
602025-01-17T21:10:57.273Z # # # ## # # # # # # #
612025-01-17T21:10:57.273Z #### # # # # # #### # #
622025-01-17T21:10:57.273Z
632025-01-17T21:10:57.273Z+ mkdir -p /var/tmp/bins
642025-01-17T21:10:57.277Z+ for t in "$input/bins/"*.gz
652025-01-17T21:10:57.277Z++ basename /input/build/work/bins/crucible-downstairs.gz
662025-01-17T21:10:57.277Z+ b=crucible-downstairs.gz
672025-01-17T21:10:57.277Z+ b=crucible-downstairs
682025-01-17T21:10:57.277Z+ gunzip
692025-01-17T21:10:59.692Z+ chmod +x /var/tmp/bins/crucible-downstairs
702025-01-17T21:10:59.695Z+ for t in "$input/bins/"*.gz
712025-01-17T21:10:59.697Z++ basename /input/build/work/bins/crucible-hammer.gz
722025-01-17T21:10:59.697Z+ b=crucible-hammer.gz
732025-01-17T21:10:59.697Z+ b=crucible-hammer
742025-01-17T21:10:59.697Z+ gunzip
752025-01-17T21:11:01.842Z+ chmod +x /var/tmp/bins/crucible-hammer
762025-01-17T21:11:01.845Z+ for t in "$input/bins/"*.gz
772025-01-17T21:11:01.847Z++ basename /input/build/work/bins/crutest.gz
782025-01-17T21:11:01.847Z+ b=crutest.gz
792025-01-17T21:11:01.847Z+ b=crutest
802025-01-17T21:11:01.847Z+ gunzip
812025-01-17T21:11:04.352Z+ chmod +x /var/tmp/bins/crutest
822025-01-17T21:11:04.354Z+ for t in "$input/bins/"*.gz
832025-01-17T21:11:04.357Z++ basename /input/build/work/bins/dsc.gz
842025-01-17T21:11:04.357Z+ b=dsc.gz
852025-01-17T21:11:04.357Z+ b=dsc
862025-01-17T21:11:04.357Z+ gunzip
872025-01-17T21:11:05.694Z+ chmod +x /var/tmp/bins/dsc
882025-01-17T21:11:05.697Z+ export BINDIR=/var/tmp/bins
892025-01-17T21:11:05.697Z+ BINDIR=/var/tmp/bins
902025-01-17T21:11:05.697Z+ jobpid=1144
912025-01-17T21:11:05.697Z+ echo 'Setup debug logging'
922025-01-17T21:11:05.701ZSetup debug logging
932025-01-17T21:11:05.701Z+ mkdir /tmp/debug
942025-01-17T21:11:05.701Z+ sleep 3600
952025-01-17T21:11:05.701Z+ psrinfo -v
962025-01-17T21:11:05.703Z+ df -h
972025-01-17T21:11:05.710Z+ prstat -d d -mLc 1
982025-01-17T21:11:05.713Z+ iostat -T d -xn 1
992025-01-17T21:11:05.713Z+ mpstat -T d 1
1002025-01-17T21:11:05.713Z+ vmstat -T d -p 1
1012025-01-17T21:11:05.713Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1022025-01-17T21:11:05.716Z+ banner test_up_2r_encrypted
1032025-01-17T21:11:05.716Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1042025-01-17T21:11:05.716Z #####
1052025-01-17T21:11:05.716Z ##### ###### #### ##### # # ##### # # #####
1062025-01-17T21:11:05.716Z # # # # # # # # # # #
1072025-01-17T21:11:05.716Z # ##### #### # # # # # ##### # #
1082025-01-17T21:11:05.716Z # # # # # # ##### # #####
1092025-01-17T21:11:05.716Z # # # # # # # # # # #
1102025-01-17T21:11:05.716Z # ###### #### # ####### #### # ####### ####### # #
1112025-01-17T21:11:05.716Z
1122025-01-17T21:11:05.716Z+ ptime -m bash /input/build/work/scripts/test_up.sh -r 2 -N encrypted
1132025-01-17T21:11:05.727Z/input/build/work
1142025-01-17T21:11:05.739ZUsing 2 region sets
1152025-01-17T21:11:05.739ZTurn off color for downstairs dump
1162025-01-17T21:11:05.754ZUpstairs using key: 3CfPMeZyN2buQCYV/Bb4kgEIGdclBr+7fVt4v3Xshto=
1172025-01-17T21:11:05.756Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1182025-01-17T21:11:05.756ZCreating 6 downstairs regions
1192025-01-17T21:11:06.114ZStarting 6 downstairs
1202025-01-17T21:11:06.116Zdsc started at PID: 1192
1212025-01-17T21:11:11.452ZDisable automatic restart on all downstairs
1222025-01-17T21:11:11.506Z
1232025-01-17T21:11:11.506ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1242025-01-17T21:11:11.510ZRunning test: span
1252025-01-17T21:11:15.687ZCompleted test: span
1262025-01-17T21:11:20.695ZRunning test: big
1272025-01-17T21:11:25.279ZCompleted test: big
1282025-01-17T21:11:30.287ZRunning test: dep
1292025-01-17T21:12:03.502ZCompleted test: dep
1302025-01-17T21:12:08.510ZRunning test: balloon
1312025-01-17T21:16:18.384ZCompleted test: balloon
1322025-01-17T21:16:23.391ZRunning test: deactivate
1332025-01-17T21:17:18.071ZCompleted test: deactivate
1342025-01-17T21:17:23.079ZRunning hammer
1352025-01-17T21:17:41.120ZRun repair tests
1362025-01-17T21:17:41.124Z/var/tmp/bins/crutest fill -g 31 -q --key 3CfPMeZyN2buQCYV/Bb4kgEIGdclBr+7fVt4v3Xshto= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1372025-01-17T21:17:41.184Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.687546926Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"task":"crutest"}
1382025-01-17T21:17:41.187Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.688160839Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"task":"crutest"}
1392025-01-17T21:17:41.187Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.689846427Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"task":"crutest"}
1402025-01-17T21:17:41.187Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.690046585Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235}
1412025-01-17T21:17:41.187Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.69009738Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235}
1422025-01-17T21:17:41.187Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.690124097Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235}
1432025-01-17T21:17:41.187Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.690142105Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235}
1442025-01-17T21:17:41.190Z{"msg":"Crucible cddd9fb8-4be3-4ffd-83d0-90280e8470a1 has session id: 86c009c9-e3af-466b-b8f5-b5f99e526d87","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.690776336Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1452025-01-17T21:17:41.190Z{"msg":"Upstairs opts: Upstairs UUID: cddd9fb8-4be3-4ffd-83d0-90280e8470a1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.690829561Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1462025-01-17T21:17:41.190Z{"msg":"Crucible stats registered with UUID: cddd9fb8-4be3-4ffd-83d0-90280e8470a1","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.690850038Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1472025-01-17T21:17:41.190Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:40.690868696Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1482025-01-17T21:17:41.190Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.692857331Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"task":"crutest"}
1492025-01-17T21:17:41.190Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.692944552Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235}
1502025-01-17T21:17:41.190Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.692987107Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235}
1512025-01-17T21:17:41.190Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.693014054Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235}
1522025-01-17T21:17:41.190Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.693029463Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235}
1532025-01-17T21:17:41.193Z{"msg":"Crucible cddd9fb8-4be3-4ffd-83d0-90280e8470a1 has session id: 9b3cd5a6-ab59-453f-84fe-2a06191def64","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.69332632Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
1542025-01-17T21:17:41.193Z{"msg":"Upstairs opts: Upstairs UUID: cddd9fb8-4be3-4ffd-83d0-90280e8470a1, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.693360447Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
1552025-01-17T21:17:41.193Z{"msg":"Crucible stats registered with UUID: cddd9fb8-4be3-4ffd-83d0-90280e8470a1","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.693376945Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
1562025-01-17T21:17:41.193Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:40.693395393Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
1572025-01-17T21:17:41.193Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.694268598Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"task":"crutest"}
1582025-01-17T21:17:41.193Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.694310124Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235}
1592025-01-17T21:17:41.193Z{"msg":"cddd9fb8-4be3-4ffd-83d0-90280e8470a1 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.694331782Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1602025-01-17T21:17:41.193Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.694357599Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"client":"0","":"downstairs","session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1612025-01-17T21:17:41.193Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.694402634Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"client":"0","":"downstairs","session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1622025-01-17T21:17:41.193Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.694428571Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"client":"1","":"downstairs","session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1632025-01-17T21:17:41.193Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.694453928Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"client":"1","":"downstairs","session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1642025-01-17T21:17:41.193Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.694496954Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"client":"2","":"downstairs","session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1652025-01-17T21:17:41.193Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.694524411Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"client":"2","":"downstairs","session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1662025-01-17T21:17:41.193Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.694557177Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1672025-01-17T21:17:41.193Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.694582494Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"io task","client":"0","":"downstairs","session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1682025-01-17T21:17:41.193Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.694635009Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"io task","client":"1","":"downstairs","session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1692025-01-17T21:17:41.193Z{"msg":"ds_connection connected from Ok(127.0.0.1:60429)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.694658936Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"io task","client":"0","":"downstairs","session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1702025-01-17T21:17:41.193Z{"msg":"ds_connection connected from Ok(127.0.0.1:34169)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.694691463Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1712025-01-17T21:17:41.193Z{"msg":"ds_connection connected from Ok(127.0.0.1:40330)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.69471427Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"io task","client":"1","":"downstairs","session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1722025-01-17T21:17:41.193Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.695481357Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"client":"0","":"downstairs","session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1732025-01-17T21:17:41.193Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.695523313Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"client":"1","":"downstairs","session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1742025-01-17T21:17:41.194Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.6955453Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"client":"2","":"downstairs","session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1752025-01-17T21:17:41.194Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.696170983Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"client":"0","":"downstairs","session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1762025-01-17T21:17:41.198Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.696322066Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"client":"1","":"downstairs","session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1772025-01-17T21:17:41.198Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.696373241Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"client":"2","":"downstairs","session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1782025-01-17T21:17:41.198Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.696536993Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1792025-01-17T21:17:41.198Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.696711214Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1802025-01-17T21:17:41.198Z{"msg":"[0]R flush_numbers: [3997, 3997, 3997, 3997, 3997]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.696852229Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"downstairs","":"downstairs","session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1812025-01-17T21:17:41.198Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.696879556Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"downstairs","":"downstairs","session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1822025-01-17T21:17:41.198Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.696894304Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"downstairs","":"downstairs","session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1832025-01-17T21:17:41.198Z{"msg":"[1]R flush_numbers: [3997, 3997, 3997, 3997, 3997]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.696907623Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"downstairs","":"downstairs","session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1842025-01-17T21:17:41.198Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.696991484Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"downstairs","":"downstairs","session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1852025-01-17T21:17:41.198Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.697040998Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"downstairs","":"downstairs","session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1862025-01-17T21:17:41.198Z{"msg":"[2]R flush_numbers: [3997, 3997, 3997, 3997, 3997]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.697108001Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"downstairs","":"downstairs","session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1872025-01-17T21:17:41.198Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.697134068Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"downstairs","":"downstairs","session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1882025-01-17T21:17:41.198Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.697149027Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"downstairs","":"downstairs","session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1892025-01-17T21:17:41.198Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.697162415Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"downstairs","":"downstairs","session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1902025-01-17T21:17:41.198Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.697175494Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"downstairs","":"downstairs","session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1912025-01-17T21:17:41.198Z{"msg":"Next flush: 3998","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.697194722Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"downstairs","":"downstairs","session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1922025-01-17T21:17:41.198Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.697227138Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"downstairs","":"downstairs","session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1932025-01-17T21:17:41.198Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.697244796Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1942025-01-17T21:17:41.198Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.697282912Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"client":"0","":"downstairs","session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1952025-01-17T21:17:41.198Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.697308259Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"client":"1","":"downstairs","session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1962025-01-17T21:17:41.198Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.697443475Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"client":"2","":"downstairs","session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1972025-01-17T21:17:41.198Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.697505438Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1982025-01-17T21:17:41.198Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.697519757Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
1992025-01-17T21:17:41.198Z{"msg":"cddd9fb8-4be3-4ffd-83d0-90280e8470a1 is now active with session: 86c009c9-e3af-466b-b8f5-b5f99e526d87","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.697557162Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
2002025-01-17T21:17:41.198Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.697575631Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"session_id":"86c009c9-e3af-466b-b8f5-b5f99e526d87"}
2012025-01-17T21:17:41.198Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.697610117Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235}
2022025-01-17T21:17:41.198Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.697630805Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235}
2032025-01-17T21:17:41.198Z{"msg":"cddd9fb8-4be3-4ffd-83d0-90280e8470a1 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.697643623Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2042025-01-17T21:17:41.198Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.697655332Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"client":"0","":"downstairs","session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2052025-01-17T21:17:41.198Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.697680079Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"client":"0","":"downstairs","session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2062025-01-17T21:17:41.198Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.697704906Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"client":"1","":"downstairs","session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2072025-01-17T21:17:41.198Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.697729124Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"client":"1","":"downstairs","session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2082025-01-17T21:17:41.198Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.697752881Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"client":"2","":"downstairs","session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2092025-01-17T21:17:41.198Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.697782708Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"client":"2","":"downstairs","session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2102025-01-17T21:17:41.198Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.697814454Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2112025-01-17T21:17:41.199Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.697834363Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"io task","client":"1","":"downstairs","session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2122025-01-17T21:17:41.199Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.69785402Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"io task","client":"0","":"downstairs","session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2132025-01-17T21:17:41.199Z{"msg":"ds_connection connected from Ok(127.0.0.1:57290)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.697872258Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"io task","client":"1","":"downstairs","session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2142025-01-17T21:17:41.199Z{"msg":"ds_connection connected from Ok(127.0.0.1:62059)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.697891696Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"io task","client":"0","":"downstairs","session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2152025-01-17T21:17:41.199Z{"msg":"ds_connection connected from Ok(127.0.0.1:55391)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.697921943Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2162025-01-17T21:17:41.199Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.69795299Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"client":"0","":"downstairs","session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2172025-01-17T21:17:41.199Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.697982016Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"client":"2","":"downstairs","session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2182025-01-17T21:17:41.199Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.698007654Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"client":"1","":"downstairs","session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2192025-01-17T21:17:41.199Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.698448166Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"client":"0","":"downstairs","session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2202025-01-17T21:17:41.199Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.698497611Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"client":"1","":"downstairs","session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2212025-01-17T21:17:41.199Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.698623267Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"client":"2","":"downstairs","session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2222025-01-17T21:17:41.199Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.69878441Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2232025-01-17T21:17:41.199Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.698915865Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2242025-01-17T21:17:41.199Z{"msg":"[0]R flush_numbers: [5318, 5318, 5318, 5317, 5315]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.698990707Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"downstairs","":"downstairs","session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2252025-01-17T21:17:41.199Z{"msg":"[0]R generation: [14, 14, 14, 14, 13]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.699018394Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"downstairs","":"downstairs","session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2262025-01-17T21:17:41.199Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.699033823Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"downstairs","":"downstairs","session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2272025-01-17T21:17:41.199Z{"msg":"[1]R flush_numbers: [5318, 5318, 5318, 5317, 5315]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.699048281Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"downstairs","":"downstairs","session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2282025-01-17T21:17:41.199Z{"msg":"[1]R generation: [14, 14, 14, 14, 13]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.699069939Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"downstairs","":"downstairs","session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2292025-01-17T21:17:41.199Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.699083537Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"downstairs","":"downstairs","session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2302025-01-17T21:17:41.199Z{"msg":"[2]R flush_numbers: [5318, 5318, 5318, 5317, 5315]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.699096346Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"downstairs","":"downstairs","session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2312025-01-17T21:17:41.199Z{Wait for a query_work_queue command to finish before sending IO
2322025-01-17T21:17:41.199Z"msg":"[2]R generation: [14, 14, 14, 14, 13]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.699124263Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"downstairs","":"downstairs","session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2332025-01-17T21:17:41.199Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.699138441Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"downstairs","":"downstairs","session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2342025-01-17T21:17:41.199Z{"msg":"Max found gen is 15","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.699158039Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"downstairs","":"downstairs","session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2352025-01-17T21:17:41.199Z{"msg":"Generation requested: 31 >= found:15","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.699171378Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"downstairs","":"downstairs","session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2362025-01-17T21:17:41.199Z{"msg":"Next flush: 5319","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.699196965Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"downstairs","":"downstairs","session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2372025-01-17T21:17:41.199Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.699209803Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"":"downstairs","":"downstairs","session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2382025-01-17T21:17:41.199Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.699230931Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2392025-01-17T21:17:41.199Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.69924296Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"client":"0","":"downstairs","session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2402025-01-17T21:17:41.199Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.699256069Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"client":"1","":"downstairs","session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2412025-01-17T21:17:41.199Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.699277546Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"client":"2","":"downstairs","session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2422025-01-17T21:17:41.199Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.699290855Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2432025-01-17T21:17:41.199Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.699314912Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2442025-01-17T21:17:41.199Z{"msg":"cddd9fb8-4be3-4ffd-83d0-90280e8470a1 is now active with session: 9b3cd5a6-ab59-453f-84fe-2a06191def64","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.699328491Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2452025-01-17T21:17:41.199Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.699347409Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235,"session_id":"9b3cd5a6-ab59-453f-84fe-2a06191def64"}
2462025-01-17T21:17:41.199Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:40.699359457Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235}
2472025-01-17T21:17:41.200ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
2482025-01-17T21:17:41.200ZFill test
2492025-01-17T21:17:41.238ZRead and Verify all blocks (0..100 range:false)
2502025-01-17T21:17:41.327ZWrote out file "/var/tmp/test_up-build/verify_file"
2512025-01-17T21:17:41.327ZCLIENT: Tests done. All submitted work has been ACK'd
2522025-01-17T21:17:41.329Z----------------------------------------------------------------
2532025-01-17T21:17:41.330Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2542025-01-17T21:17:41.330ZStates: Active Active Active
2552025-01-17T21:17:41.330ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2562025-01-17T21:17:41.330Z 1002 Acked Read 50 Done Done Done false
2572025-01-17T21:17:41.330Z STATES DS:0 DS:1 DS:2 TOTAL
2582025-01-17T21:17:41.330Z Sent 0 0 0 0
2592025-01-17T21:17:41.330Z Done 1 1 1 3
2602025-01-17T21:17:41.330Z Skipped 0 0 0 0
2612025-01-17T21:17:41.330Z Error 0 0 0 0
2622025-01-17T21:17:41.330ZLast Flush: 1001 1001 1001
2632025-01-17T21:17:41.330ZDownstairs last five completed: 1001 1000
2642025-01-17T21:17:41.330ZUpstairs last five completed: 1002 1001 1000
2652025-01-17T21:17:41.330Z----------------------------------------------------------------
2662025-01-17T21:17:41.330Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2672025-01-17T21:17:41.330ZStates: Active Active Active
2682025-01-17T21:17:41.330ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2692025-01-17T21:17:41.330Z 1002 Acked Read 50 Done Done Done false
2702025-01-17T21:17:41.330Z STATES DS:0 DS:1 DS:2 TOTAL
2712025-01-17T21:17:41.330Z Sent 0 0 0 0
2722025-01-17T21:17:41.330Z Done 1 1 1 3
2732025-01-17T21:17:41.330Z Skipped 0 0 0 0
2742025-01-17T21:17:41.330Z Error 0 0 0 0
2752025-01-17T21:17:41.330ZLast Flush: 1001 1001 1001
2762025-01-17T21:17:41.330ZDownstairs last five completed: 1001 1000
2772025-01-17T21:17:41.330ZUpstairs last five completed: 1002 1001 1000
2782025-01-17T21:17:41.330ZCLIENT: Up:0 ds:2 act:6
2792025-01-17T21:17:45.330Z----------------------------------------------------------------
2802025-01-17T21:17:45.330Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2812025-01-17T21:17:45.330ZDownstairs last five completed: 1003 1002 1001 1000
2822025-01-17T21:17:45.330ZUpstairs last five completed: 1003 1002 1001 1000
2832025-01-17T21:17:45.333Z----------------------------------------------------------------
2842025-01-17T21:17:45.333Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2852025-01-17T21:17:45.333ZDownstairs last five completed: 1003 1002 1001 1000
2862025-01-17T21:17:45.333ZUpstairs last five completed: 1003 1002 1001 1000
2872025-01-17T21:17:45.333ZCLIENT: Up:0 ds:0 act:6
2882025-01-17T21:17:45.333ZCLIENT: All crucible jobs finished, exiting program
2892025-01-17T21:17:45.333Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:44.833025874Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1235}
2902025-01-17T21:17:45.342ZRepair setup passed
2912025-01-17T21:17:45.345ZCopy the region for /var/tmp/test_up-build/8830
2922025-01-17T21:17:45.348Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2932025-01-17T21:17:45.350Z/var/tmp/bins/crutest fill -g 32 -q --key 3CfPMeZyN2buQCYV/Bb4kgEIGdclBr+7fVt4v3Xshto= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2942025-01-17T21:17:45.409Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.911068997Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"task":"crutest"}
2952025-01-17T21:17:45.412Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.91159294Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"task":"crutest"}
2962025-01-17T21:17:45.412Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.913246281Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"task":"crutest"}
2972025-01-17T21:17:45.412Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.91344041Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241}
2982025-01-17T21:17:45.412Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.913491175Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241}
2992025-01-17T21:17:45.412Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.913519351Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241}
3002025-01-17T21:17:45.412Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.91353798Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241}
3012025-01-17T21:17:45.412Z{"msg":"Crucible 7c7dbf08-5683-439b-99a2-58424b7077d0 has session id: 7a59a558-1472-4500-8aeb-1e47442ee7d9","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.914212027Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3022025-01-17T21:17:45.412Z{"msg":"Upstairs opts: Upstairs UUID: 7c7dbf08-5683-439b-99a2-58424b7077d0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.914248423Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3032025-01-17T21:17:45.412Z{"msg":"Crucible stats registered with UUID: 7c7dbf08-5683-439b-99a2-58424b7077d0","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.91426845Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3042025-01-17T21:17:45.412Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:44.914299177Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3052025-01-17T21:17:45.416Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.916285042Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"task":"crutest"}
3062025-01-17T21:17:45.416Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.916372463Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241}
3072025-01-17T21:17:45.416Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.916410069Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241}
3082025-01-17T21:17:45.416Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.916440705Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241}
3092025-01-17T21:17:45.416Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.916463373Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241}
3102025-01-17T21:17:45.416Z{"msg":"Crucible 7c7dbf08-5683-439b-99a2-58424b7077d0 has session id: b0d5d910-2288-4f31-86cc-00dfa1ef89cd","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.916788128Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3112025-01-17T21:17:45.416Z{"msg":"Upstairs opts: Upstairs UUID: 7c7dbf08-5683-439b-99a2-58424b7077d0, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.916847481Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3122025-01-17T21:17:45.416Z{"msg":"Crucible stats registered with UUID: 7c7dbf08-5683-439b-99a2-58424b7077d0","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.916879758Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3132025-01-17T21:17:45.416Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:44.916897466Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3142025-01-17T21:17:45.416Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.917567193Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"task":"crutest"}
3152025-01-17T21:17:45.416Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.917592221Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241}
3162025-01-17T21:17:45.416Z{"msg":"7c7dbf08-5683-439b-99a2-58424b7077d0 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.917668262Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3172025-01-17T21:17:45.416Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.917723676Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"client":"0","":"downstairs","session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3182025-01-17T21:17:45.416Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.91778107Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"client":"0","":"downstairs","session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3192025-01-17T21:17:45.416Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.917818446Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"client":"1","":"downstairs","session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3202025-01-17T21:17:45.416Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.917844234Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"client":"1","":"downstairs","session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3212025-01-17T21:17:45.416Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.917885409Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"client":"2","":"downstairs","session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3222025-01-17T21:17:45.416Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.917912356Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"client":"2","":"downstairs","session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3232025-01-17T21:17:45.416Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.917946032Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"io task","client":"2","":"downstairs","session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3242025-01-17T21:17:45.416Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.917972689Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"io task","client":"0","":"downstairs","session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3252025-01-17T21:17:45.421Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.918202795Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"io task","client":"1","":"downstairs","session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3262025-01-17T21:17:45.421Z{"msg":"ds_connection connected from Ok(127.0.0.1:46517)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.91825023Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"io task","client":"2","":"downstairs","session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3272025-01-17T21:17:45.421Z{"msg":"ds_connection connected from Ok(127.0.0.1:50967)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.918287206Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"io task","client":"1","":"downstairs","session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3282025-01-17T21:17:45.421Z{"msg":"ds_connection connected from Ok(127.0.0.1:41930)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.91833554Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"io task","client":"0","":"downstairs","session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3292025-01-17T21:17:45.421Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.91870774Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"client":"1","":"downstairs","session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3302025-01-17T21:17:45.421Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.918747026Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"client":"2","":"downstairs","session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3312025-01-17T21:17:45.421Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.918762574Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"client":"0","":"downstairs","session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3322025-01-17T21:17:45.421Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.919334642Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"client":"0","":"downstairs","session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3332025-01-17T21:17:45.422Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.919415393Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"client":"1","":"downstairs","session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3342025-01-17T21:17:45.422Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.91944875Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"client":"2","":"downstairs","session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3352025-01-17T21:17:45.422Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.919743248Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3362025-01-17T21:17:45.422Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.919787813Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3372025-01-17T21:17:45.422Z{"msg":"[0]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.919835808Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"downstairs","":"downstairs","session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3382025-01-17T21:17:45.422Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.919859675Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"downstairs","":"downstairs","session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3392025-01-17T21:17:45.422Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.919913039Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"downstairs","":"downstairs","session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3402025-01-17T21:17:45.422Z{"msg":"[1]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.919938287Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"downstairs","":"downstairs","session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3412025-01-17T21:17:45.422Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.919966054Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"downstairs","":"downstairs","session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3422025-01-17T21:17:45.422Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.919989641Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"downstairs","":"downstairs","session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3432025-01-17T21:17:45.422Z{"msg":"[2]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.920012919Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"downstairs","":"downstairs","session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3442025-01-17T21:17:45.422Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.920034476Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"downstairs","":"downstairs","session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3452025-01-17T21:17:45.422Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.920056904Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"downstairs","":"downstairs","session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3462025-01-17T21:17:45.422Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.920076352Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"downstairs","":"downstairs","session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3472025-01-17T21:17:45.422Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.92009693Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"downstairs","":"downstairs","session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3482025-01-17T21:17:45.422Z{"msg":"Next flush: 3999","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.920136235Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"downstairs","":"downstairs","session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3492025-01-17T21:17:45.422Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.920167612Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"downstairs","":"downstairs","session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3502025-01-17T21:17:45.422Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.920197909Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3512025-01-17T21:17:45.422Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.920211607Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"client":"0","":"downstairs","session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3522025-01-17T21:17:45.422Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.920229325Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"client":"1","":"downstairs","session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3532025-01-17T21:17:45.422Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.920266661Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"client":"2","":"downstairs","session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3542025-01-17T21:17:45.422Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.920451491Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3552025-01-17T21:17:45.422Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.920479558Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3562025-01-17T21:17:45.422Z{"msg":"7c7dbf08-5683-439b-99a2-58424b7077d0 is now active with session: 7a59a558-1472-4500-8aeb-1e47442ee7d9","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.920506535Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3572025-01-17T21:17:45.422Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.920571988Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"session_id":"7a59a558-1472-4500-8aeb-1e47442ee7d9"}
3582025-01-17T21:17:45.422Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.920602615Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241}
3592025-01-17T21:17:45.423Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.920631732Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241}
3602025-01-17T21:17:45.423Z{"msg":"7c7dbf08-5683-439b-99a2-58424b7077d0 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.9206477Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3612025-01-17T21:17:45.423Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.920687496Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"client":"0","":"downstairs","session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3622025-01-17T21:17:45.423Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.920726782Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"client":"0","":"downstairs","session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3632025-01-17T21:17:45.423Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.920762758Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"client":"1","":"downstairs","session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3642025-01-17T21:17:45.423Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.920799974Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"client":"1","":"downstairs","session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3652025-01-17T21:17:45.423Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.92083321Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"client":"2","":"downstairs","session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3662025-01-17T21:17:45.423Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.920858787Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"client":"2","":"downstairs","session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3672025-01-17T21:17:45.423Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.920882655Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"io task","client":"2","":"downstairs","session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3682025-01-17T21:17:45.423Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.920907082Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"io task","client":"0","":"downstairs","session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3692025-01-17T21:17:45.423Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.920931809Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"io task","client":"1","":"downstairs","session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3702025-01-17T21:17:45.423Z{"msg":"ds_connection connected from Ok(127.0.0.1:58595)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.920955067Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"io task","client":"1","":"downstairs","session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3712025-01-17T21:17:45.423Z{"msg":"ds_connection connected from Ok(127.0.0.1:52252)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.920977494Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"io task","client":"0","":"downstairs","session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3722025-01-17T21:17:45.423Z{"msg":"ds_connection connected from Ok(127.0.0.1:36970)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.92101589Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"io task","client":"2","":"downstairs","session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3732025-01-17T21:17:45.423Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.921031798Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"client":"0","":"downstairs","session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3742025-01-17T21:17:45.423Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.921045897Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"client":"1","":"downstairs","session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3752025-01-17T21:17:45.423Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.921059135Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"client":"2","":"downstairs","session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3762025-01-17T21:17:45.423Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.921251505Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"client":"0","":"downstairs","session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3772025-01-17T21:17:45.423Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.921457832Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"client":"1","":"downstairs","session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3782025-01-17T21:17:45.423Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.921515096Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"client":"2","":"downstairs","session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3792025-01-17T21:17:45.423Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.921610536Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3802025-01-17T21:17:45.423Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.921802925Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3812025-01-17T21:17:45.423Z{"msg":"[0]R flush_numbers: [5319, 5319, 5319, 5319, 5319]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.921839011Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"downstairs","":"downstairs","session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3822025-01-17T21:17:45.423Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.921855649Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"downstairs","":"downstairs","session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3832025-01-17T21:17:45.423Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.921869438Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"downstairs","":"downstairs","session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3842025-01-17T21:17:45.423Z{"msg":"[1]R flush_numbers: [5319, 5319, 5319, 5319, 5319]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.921890525Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"downstairs","":"downstairs","session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3852025-01-17T21:17:45.423Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.921903864Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"downstairs","":"downstairs","session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3862025-01-17T21:17:45.424Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.921926761Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"downstairs","":"downstairs","session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3872025-01-17T21:17:45.424Z{"msg":"[2]R flush_numbers: [5319, 5319, 5319, 5319, 5319]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
3882025-01-17T21:17:45.424Z,"time":"2025-01-17T21:17:44.921949399Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"downstairs","":"downstairs","session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3892025-01-17T21:17:45.424Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.921981206Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"downstairs","":"downstairs","session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3902025-01-17T21:17:45.424Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.921994064Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"downstairs","":"downstairs","session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3912025-01-17T21:17:45.424Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.922014872Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"downstairs","":"downstairs","session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3922025-01-17T21:17:45.424Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.922027971Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"downstairs","":"downstairs","session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3932025-01-17T21:17:45.424Z{"msg":"Next flush: 5320","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.922040279Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"downstairs","":"downstairs","session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3942025-01-17T21:17:45.424Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.922060677Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"":"downstairs","":"downstairs","session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3952025-01-17T21:17:45.424Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.922073676Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3962025-01-17T21:17:45.424Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.922085324Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"client":"0","":"downstairs","session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3972025-01-17T21:17:45.424Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.922105052Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"client":"1","":"downstairs","session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3982025-01-17T21:17:45.424Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.922118231Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"client":"2","":"downstairs","session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
3992025-01-17T21:17:45.424Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.922130779Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
4002025-01-17T21:17:45.424Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.922142968Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
4012025-01-17T21:17:45.424Z{"msg":"7c7dbf08-5683-439b-99a2-58424b7077d0 is now active with session: b0d5d910-2288-4f31-86cc-00dfa1ef89cd","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.922162306Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
4022025-01-17T21:17:45.424Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.922174335Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241,"session_id":"b0d5d910-2288-4f31-86cc-00dfa1ef89cd"}
4032025-01-17T21:17:45.424Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:44.922186024Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241}
4042025-01-17T21:17:45.425ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
4052025-01-17T21:17:45.425ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4062025-01-17T21:17:45.425ZFill test
4072025-01-17T21:17:45.461ZRead and Verify all blocks (0..100 range:false)
4082025-01-17T21:17:45.539ZWrote out file "/var/tmp/test_up-build/verify_file"
4092025-01-17T21:17:45.540ZCLIENT: Tests done. All submitted work has been ACK'd
4102025-01-17T21:17:45.543Z----------------------------------------------------------------
4112025-01-17T21:17:45.543Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4122025-01-17T21:17:45.543ZStates: Active Active Active
4132025-01-17T21:17:45.543ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4142025-01-17T21:17:45.543Z 1002 Acked Read 50 Done Done Done false
4152025-01-17T21:17:45.543Z STATES DS:0 DS:1 DS:2 TOTAL
4162025-01-17T21:17:45.543Z Sent 0 0 0 0
4172025-01-17T21:17:45.543Z Done 1 1 1 3
4182025-01-17T21:17:45.543Z Skipped 0 0 0 0
4192025-01-17T21:17:45.543Z Error 0 0 0 0
4202025-01-17T21:17:45.544ZLast Flush: 1001 1001 1001
4212025-01-17T21:17:45.544ZDownstairs last five completed: 1001 1000
4222025-01-17T21:17:45.544ZUpstairs last five completed: 1002 1001 1000
4232025-01-17T21:17:45.544Z----------------------------------------------------------------
4242025-01-17T21:17:45.544Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4252025-01-17T21:17:45.544ZStates: Active Active Active
4262025-01-17T21:17:45.544ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4272025-01-17T21:17:45.544Z 1002 Acked Read 50 Done Done Done false
4282025-01-17T21:17:45.544Z STATES DS:0 DS:1 DS:2 TOTAL
4292025-01-17T21:17:45.544Z Sent 0 0 0 0
4302025-01-17T21:17:45.544Z Done 1 1 1 3
4312025-01-17T21:17:45.544Z Skipped 0 0 0 0
4322025-01-17T21:17:45.544Z Error 0 0 0 0
4332025-01-17T21:17:45.544ZLast Flush: 1001 1001 1001
4342025-01-17T21:17:45.544ZDownstairs last five completed: 1001 1000
4352025-01-17T21:17:45.544ZUpstairs last five completed: 1002 1001 1000
4362025-01-17T21:17:45.544ZCLIENT: Up:0 ds:2 act:6
4372025-01-17T21:17:49.541Z----------------------------------------------------------------
4382025-01-17T21:17:49.541Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4392025-01-17T21:17:49.541ZDownstairs last five completed: 1003 1002 1001 1000
4402025-01-17T21:17:49.541ZUpstairs last five completed: 1003 1002 1001 1000
4412025-01-17T21:17:49.543Z----------------------------------------------------------------
4422025-01-17T21:17:49.543Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4432025-01-17T21:17:49.543ZDownstairs last five completed: 1003 1002 1001 1000
4442025-01-17T21:17:49.543ZUpstairs last five completed: 1003 1002 1001 1000
4452025-01-17T21:17:49.543ZCLIENT: Up:0 ds:0 act:6
4462025-01-17T21:17:49.543ZCLIENT: All crucible jobs finished, exiting program
4472025-01-17T21:17:49.543Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:49.043224487Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1241}
4482025-01-17T21:17:49.549ZRepair part 1 passed
4492025-01-17T21:17:49.551Z
4502025-01-17T21:17:49.551ZKill the current downstairs
4512025-01-17T21:17:49.717ZDownstairs 2 stopped
4522025-01-17T21:17:49.717Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
4532025-01-17T21:17:49.720ZNow put back the original so we have a mismatch
4542025-01-17T21:17:49.720Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
4552025-01-17T21:17:49.723ZRestart downstairs with old directory
4562025-01-17T21:17:49.776Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4572025-01-17T21:17:49.792ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4582025-01-17T21:17:49.795Z 0 000-009 32 32 31 3999 3999 3998 F F F <---
4592025-01-17T21:17:49.795Z 1 010-019 32 32 31 3999 3999 3998 F F F <---
4602025-01-17T21:17:49.795Z 2 020-029 32 32 31 3999 3999 3998 F F F <---
4612025-01-17T21:17:49.795Z 3 030-039 32 32 31 3999 3999 3998 F F F <---
4622025-01-17T21:17:49.795Z 4 040-049 32 32 31 3999 3999 3998 F F F <---
4632025-01-17T21:17:49.795ZMax gen: 32, Max flush: 3999
4642025-01-17T21:17:49.795ZError: Difference in extent metadata found!
4652025-01-17T21:17:49.797Zdump test found error as expected
4662025-01-17T21:17:49.797Z
4672025-01-17T21:17:49.798Z
4682025-01-17T21:17:49.798Z/var/tmp/bins/crutest verify --range -g 33 -q --key 3CfPMeZyN2buQCYV/Bb4kgEIGdclBr+7fVt4v3Xshto= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
4692025-01-17T21:17:49.854Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.355956559Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"task":"crutest"}
4702025-01-17T21:17:49.857Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.356469893Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"task":"crutest"}
4712025-01-17T21:17:49.857Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.358094907Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"task":"crutest"}
4722025-01-17T21:17:49.857Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.358273158Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249}
4732025-01-17T21:17:49.857Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.35835192Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249}
4742025-01-17T21:17:49.857Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.358389476Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249}
4752025-01-17T21:17:49.857Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.358421742Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249}
4762025-01-17T21:17:49.862Z{"msg":"Crucible 2c18d9ad-359e-48c1-af68-791c7e0a33ad has session id: 9b289e93-612f-4ba0-ae9b-3a16e0fc0e38","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.358863824Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
4772025-01-17T21:17:49.862Z{"msg":"Upstairs opts: Upstairs UUID: 2c18d9ad-359e-48c1-af68-791c7e0a33ad, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.358913039Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
4782025-01-17T21:17:49.862Z{"msg":"Crucible stats registered with UUID: 2c18d9ad-359e-48c1-af68-791c7e0a33ad","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.358938506Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
4792025-01-17T21:17:49.862Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:49.358971773Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
4802025-01-17T21:17:49.862Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.360984535Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"task":"crutest"}
4812025-01-17T21:17:49.862Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.361066916Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249}
4822025-01-17T21:17:49.862Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"abd5430f1508af0bc008143ea2566277ef714ec2\",\n git_commit_timestamp: \"2025-01-17T20:55:03.000000000Z\",\n git_branch: \"alan/tokio-to-140\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.361104042Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249}
4832025-01-17T21:17:49.862Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.36111886Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249}
4842025-01-17T21:17:49.862Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.361140278Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249}
4852025-01-17T21:17:49.862Z{"msg":"Crucible 2c18d9ad-359e-48c1-af68-791c7e0a33ad has session id: aa6878ec-ad1d-4c83-9d81-ab18a18fef1b","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.361527516Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
4862025-01-17T21:17:49.862Z{"msg":"Upstairs opts: Upstairs UUID: 2c18d9ad-359e-48c1-af68-791c7e0a33ad, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.361575811Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
4872025-01-17T21:17:49.862Z{"msg":"Crucible stats registered with UUID: 2c18d9ad-359e-48c1-af68-791c7e0a33ad","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.361600178Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
4882025-01-17T21:17:49.862Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:49.361631055Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
4892025-01-17T21:17:49.862Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.362238489Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"task":"crutest"}
4902025-01-17T21:17:49.862Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.362289593Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249}
4912025-01-17T21:17:49.862Z{"msg":"2c18d9ad-359e-48c1-af68-791c7e0a33ad active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.362326879Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
4922025-01-17T21:17:49.865Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.362351487Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"0","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
4932025-01-17T21:17:49.865Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.362397122Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"0","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
4942025-01-17T21:17:49.865Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.362425189Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"1","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
4952025-01-17T21:17:49.865Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.362445586Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"1","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
4962025-01-17T21:17:49.865Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.362479793Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"2","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
4972025-01-17T21:17:49.865Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.36250389Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"2","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
4982025-01-17T21:17:49.865Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.362549125Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"io task","client":"2","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
4992025-01-17T21:17:49.865Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.362572013Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"io task","client":"0","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5002025-01-17T21:17:49.865Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.362603779Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"io task","client":"1","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5012025-01-17T21:17:49.865Z{"msg":"ds_connection connected from Ok(127.0.0.1:36802)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.362627007Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"io task","client":"1","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5022025-01-17T21:17:49.865Z{"msg":"ds_connection connected from Ok(127.0.0.1:38153)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.362708258Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"io task","client":"0","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5032025-01-17T21:17:49.865Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:49.362789669Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"io task","client":"2","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5042025-01-17T21:17:49.865Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:49.362880649Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"io task","client":"2","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5052025-01-17T21:17:49.865Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:17:49.362921465Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5062025-01-17T21:17:49.865Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.362990327Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"io task","client":"2","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5072025-01-17T21:17:49.866Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.363318182Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"0","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5082025-01-17T21:17:49.866Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.363502492Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"1","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5092025-01-17T21:17:49.866Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.363899459Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"0","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5102025-01-17T21:17:49.866Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.364006997Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"1","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5112025-01-17T21:17:49.866Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.364227764Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5122025-01-17T21:17:49.866Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:49.364275808Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5132025-01-17T21:17:59.863Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.363906229Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"io task","client":"2","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5142025-01-17T21:17:59.866Z{"msg":"ds_connection connected from Ok(127.0.0.1:54588)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.364089709Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"io task","client":"2","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5152025-01-17T21:17:59.867Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.365427994Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"2","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5162025-01-17T21:17:59.867Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.366054106Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"2","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5172025-01-17T21:17:59.867Z{"msg":"[0]R flush_numbers: [3999, 3999, 3999, 3999, 3999]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.366513377Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5182025-01-17T21:17:59.867Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.366541763Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5192025-01-17T21:17:59.867Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.366556802Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5202025-01-17T21:17:59.867Z{"msg":"[1]R flush_numbers: [3999, 3999, 3999, 3999, 3999]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.36657741Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5212025-01-17T21:17:59.867Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.366591978Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5222025-01-17T21:17:59.867Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.366612666Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5232025-01-17T21:17:59.867Z{"msg":"[2]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.366626354Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5242025-01-17T21:17:59.867Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.366645982Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5252025-01-17T21:17:59.867Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.366659361Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5262025-01-17T21:17:59.867Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.366678509Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5272025-01-17T21:17:59.867Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.366691747Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5282025-01-17T21:17:59.867Z{"msg":"Next flush: 4000","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.366711755Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5292025-01-17T21:17:59.867Z{"msg":"Extent 0 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.366725314Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5302025-01-17T21:17:59.867Z{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.366745391Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5312025-01-17T21:17:59.867Z{"msg":"extent:0 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.36675973Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5322025-01-17T21:17:59.867Z{"msg":"extent:0 flush: 3999 3999 3998 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.366781028Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5332025-01-17T21:17:59.868Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.366795416Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5342025-01-17T21:17:59.868Z{"msg":"extent:0 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.366815984Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5352025-01-17T21:17:59.868Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.366830512Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5362025-01-17T21:17:59.868Z{"msg":"find dest for source 0 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.366858279Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5372025-01-17T21:17:59.868Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.366891345Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5382025-01-17T21:17:59.872Z{"msg":"Extent 1 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.366927812Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5392025-01-17T21:17:59.872Z{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.366960418Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5402025-01-17T21:17:59.872Z{"msg":"extent:1 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.366985795Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5412025-01-17T21:17:59.872Z{"msg":"extent:1 flush: 3999 3999 3998 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.367041169Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5422025-01-17T21:17:59.872Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.367072636Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5432025-01-17T21:17:59.872Z{"msg":"extent:1 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.367098623Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5442025-01-17T21:17:59.872Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.367132359Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5452025-01-17T21:17:59.872Z{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.367158047Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5462025-01-17T21:17:59.872Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.367186084Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5472025-01-17T21:17:59.872Z{"msg":"Extent 2 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.367211031Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5482025-01-17T21:17:59.872Z{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.367243227Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5492025-01-17T21:17:59.872Z{"msg":"extent:2 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.367275764Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5502025-01-17T21:17:59.872Z{"msg":"extent:2 flush: 3999 3999 3998 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.36730856Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5512025-01-17T21:17:59.872Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.367341787Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5522025-01-17T21:17:59.872Z{"msg":"extent:2 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.367367364Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5532025-01-17T21:17:59.872Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.367398691Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5542025-01-17T21:17:59.872Z{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.367424068Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5552025-01-17T21:17:59.872Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.367456404Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5562025-01-17T21:17:59.872Z{"msg":"Extent 3 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.367482152Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5572025-01-17T21:17:59.872Z{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.367505189Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5582025-01-17T21:17:59.872Z{"msg":"extent:3 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.367536936Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5592025-01-17T21:17:59.872Z{"msg":"extent:3 flush: 3999 3999 3998 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.367563063Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5602025-01-17T21:17:59.872Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.36758812Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5612025-01-17T21:17:59.872Z{"msg":"extent:3 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.367613138Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5622025-01-17T21:17:59.872Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.367647744Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5632025-01-17T21:17:59.872Z{"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.367673271Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5642025-01-17T21:17:59.872Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.367705937Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5652025-01-17T21:17:59.872Z{"msg":"Extent 4 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.367732765Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5662025-01-17T21:17:59.872Z{"msg":"First source client ID for extent 4","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.367764141Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5672025-01-17T21:17:59.872Z{"msg":"extent:4 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.367788879Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5682025-01-17T21:17:59.872Z{"msg":"extent:4 flush: 3999 3999 3998 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.367816696Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5692025-01-17T21:17:59.872Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.367841493Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5702025-01-17T21:17:59.872Z{"msg":"extent:4 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.36786864Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5712025-01-17T21:17:59.872Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.367893717Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5722025-01-17T21:17:59.872Z{"msg":"find dest for source 0 for extent at index 4","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.367921214Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5732025-01-17T21:17:59.872Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.367945422Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5742025-01-17T21:17:59.872Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.367974468Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"0","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5752025-01-17T21:17:59.873Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.367998166Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"1","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5762025-01-17T21:17:59.873Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.368025413Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"2","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5772025-01-17T21:17:59.873Z{"msg":"Full repair list: {ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.36805613Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5782025-01-17T21:17:59.873Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 4000, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 4000, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 4000, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 4000, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 4000, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.368167087Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5792025-01-17T21:17:59.873Z{"msg":"starting reconciliation 408dd8fa-eb03-4e31-9c0d-4bb032173385: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.368204114Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5802025-01-17T21:17:59.873Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.368228191Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"2","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5812025-01-17T21:17:59.876Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.372571151Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"2","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5822025-01-17T21:17:59.876Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.376524053Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"2","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5832025-01-17T21:17:59.879Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.380240641Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"2","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5842025-01-17T21:17:59.883Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.383798406Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"2","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5852025-01-17T21:17:59.886Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.386787092Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"downstairs","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5862025-01-17T21:17:59.886Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.386823418Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"0","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5872025-01-17T21:17:59.886Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.386839966Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"1","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5882025-01-17T21:17:59.886Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.386853825Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"2","":"downstairs","session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5892025-01-17T21:17:59.889Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.386877782Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5902025-01-17T21:17:59.889Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.38690299Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5912025-01-17T21:17:59.889Z{"msg":"2c18d9ad-359e-48c1-af68-791c7e0a33ad is now active with session: 9b289e93-612f-4ba0-ae9b-3a16e0fc0e38","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.386916858Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"session_id":"9b289e93-612f-4ba0-ae9b-3a16e0fc0e38"}
5922025-01-17T21:17:59.889Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.386929027Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249}
5932025-01-17T21:17:59.889Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.386951644Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249}
5942025-01-17T21:17:59.889Z{"msg":"2c18d9ad-359e-48c1-af68-791c7e0a33ad active request set","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.386968452Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
5952025-01-17T21:17:59.889Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.38698733Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"0","":"downstairs","session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
5962025-01-17T21:17:59.889Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.387046984Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"0","":"downstairs","session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
5972025-01-17T21:17:59.889Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.387092989Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"1","":"downstairs","session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
5982025-01-17T21:17:59.889Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.387122976Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"1","":"downstairs","session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
5992025-01-17T21:17:59.889Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.387151813Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"2","":"downstairs","session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6002025-01-17T21:17:59.889Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.38718009Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"2","":"downstairs","session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6012025-01-17T21:17:59.889Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.387206907Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"io task","client":"0","":"downstairs","session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6022025-01-17T21:17:59.889Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.387237374Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"io task","client":"2","":"downstairs","session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6032025-01-17T21:17:59.889Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.38726614Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"io task","client":"1","":"downstairs","session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6042025-01-17T21:17:59.889Z{"msg":"ds_connection connected from Ok(127.0.0.1:54618)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.387295607Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"io task","client":"0","":"downstairs","session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6052025-01-17T21:17:59.889Z{"msg":"ds_connection connected from Ok(127.0.0.1:41233)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.387325604Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"io task","client":"2","":"downstairs","session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6062025-01-17T21:17:59.890Z{"msg":"ds_connection connected from Ok(127.0.0.1:44683)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.387351541Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"io task","client":"1","":"downstairs","session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6072025-01-17T21:17:59.890Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.38763949Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"0","":"downstairs","session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6082025-01-17T21:17:59.890Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.387772656Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"1","":"downstairs","session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6092025-01-17T21:17:59.890Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.387881334Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"2","":"downstairs","session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6102025-01-17T21:17:59.890Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.388368961Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"1","":"downstairs","session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6112025-01-17T21:17:59.890Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.388407177Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"0","":"downstairs","session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6122025-01-17T21:17:59.890Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.388434184Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"2","":"downstairs","session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6132025-01-17T21:17:59.890Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.388762998Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6142025-01-17T21:17:59.890Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.388809143Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6152025-01-17T21:17:59.890Z{"msg":"[0]R flush_numbers: [5320, 5320, 5320, 5320, 5320]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.388825562Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"downstairs","":"downstairs","session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6162025-01-17T21:17:59.890Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.388847129Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"downstairs","":"downstairs","session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6172025-01-17T21:17:59.890Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.388863038Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"downstairs","":"downstairs","session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6182025-01-17T21:17:59.890Z{"msg":"[1]R flush_numbers: [5320, 5320, 5320, 5320, 5320]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
6192025-01-17T21:17:59.890Z,"time":"2025-01-17T21:17:59.388896224Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"downstairs","":"downstairs","session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6202025-01-17T21:17:59.890Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.388922331Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"downstairs","":"downstairs","session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6212025-01-17T21:17:59.890Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.38893621Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"downstairs","":"downstairs","session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6222025-01-17T21:17:59.890Z{"msg":"[2]R flush_numbers: [5320, 5320, 5320, 5320, 5320]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.388949848Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"downstairs","":"downstairs","session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6232025-01-17T21:17:59.890Z{"msg":"[2]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.388982135Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"downstairs","":"downstairs","session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6242025-01-17T21:17:59.890Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.389007082Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"downstairs","":"downstairs","session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6252025-01-17T21:17:59.890Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.38902748Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"downstairs","":"downstairs","session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6262025-01-17T21:17:59.890Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.389040468Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"downstairs","":"downstairs","session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6272025-01-17T21:17:59.890Z{"msg":"Next flush: 5321","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.389059636Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"downstairs","":"downstairs","session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6282025-01-17T21:17:59.890Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.389072615Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"":"downstairs","":"downstairs","session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6292025-01-17T21:17:59.890Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.38912155Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6302025-01-17T21:17:59.890Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.389136738Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"0","":"downstairs","session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6312025-01-17T21:17:59.890Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.389150496Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"1","":"downstairs","session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6322025-01-17T21:17:59.890Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.389170744Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"client":"2","":"downstairs","session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6332025-01-17T21:17:59.890Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.389184053Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6342025-01-17T21:17:59.890Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.389202471Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6352025-01-17T21:17:59.890Z{"msg":"2c18d9ad-359e-48c1-af68-791c7e0a33ad is now active with session: aa6878ec-ad1d-4c83-9d81-ab18a18fef1b","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.389215169Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6362025-01-17T21:17:59.890Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.389228418Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249,"session_id":"aa6878ec-ad1d-4c83-9d81-ab18a18fef1b"}
6372025-01-17T21:17:59.890Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:17:59.389240617Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249}
6382025-01-17T21:17:59.890ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
6392025-01-17T21:17:59.890ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
6402025-01-17T21:17:59.890ZRead and Verify all blocks (0..100 range:true)
6412025-01-17T21:17:59.966ZWrote out file "/var/tmp/test_up-build/verify_file"
6422025-01-17T21:17:59.966ZVerify test completed
6432025-01-17T21:17:59.969ZWrote out file "/var/tmp/test_up-build/verify_file"
6442025-01-17T21:17:59.969ZCLIENT: Tests done. All submitted work has been ACK'd
6452025-01-17T21:17:59.969Z----------------------------------------------------------------
6462025-01-17T21:17:59.969Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6472025-01-17T21:17:59.969ZStates: Active Active Active
6482025-01-17T21:17:59.969ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6492025-01-17T21:17:59.969Z 1000 Acked Read 50 Done Done Done false
6502025-01-17T21:17:59.969Z STATES DS:0 DS:1 DS:2 TOTAL
6512025-01-17T21:17:59.969Z Sent 0 0 0 0
6522025-01-17T21:17:59.969Z Done 1 1 1 3
6532025-01-17T21:17:59.969Z Skipped 0 0 0 0
6542025-01-17T21:17:59.969Z Error 0 0 0 0
6552025-01-17T21:17:59.969ZLast Flush: 0 0 0
6562025-01-17T21:17:59.969ZDownstairs last five completed:
6572025-01-17T21:17:59.969ZUpstairs last five completed: 1000
6582025-01-17T21:17:59.969Z----------------------------------------------------------------
6592025-01-17T21:17:59.969Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6602025-01-17T21:17:59.970ZStates: Active Active Active
6612025-01-17T21:17:59.970ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6622025-01-17T21:17:59.970Z 1000 Acked Read 50 Done Done Done false
6632025-01-17T21:17:59.970Z STATES DS:0 DS:1 DS:2 TOTAL
6642025-01-17T21:17:59.970Z Sent 0 0 0 0
6652025-01-17T21:17:59.970Z Done 1 1 1 3
6662025-01-17T21:17:59.970Z Skipped 0 0 0 0
6672025-01-17T21:17:59.970Z Error 0 0 0 0
6682025-01-17T21:17:59.970ZLast Flush: 0 0 0
6692025-01-17T21:17:59.970ZDownstairs last five completed:
6702025-01-17T21:17:59.970ZUpstairs last five completed: 1000
6712025-01-17T21:17:59.970ZCLIENT: Up:0 ds:2 act:6
6722025-01-17T21:18:03.970Z----------------------------------------------------------------
6732025-01-17T21:18:03.970Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6742025-01-17T21:18:03.970ZDownstairs last five completed: 1001 1000
6752025-01-17T21:18:03.970ZUpstairs last five completed: 1001 1000
6762025-01-17T21:18:03.974Z----------------------------------------------------------------
6772025-01-17T21:18:03.974Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6782025-01-17T21:18:03.974ZDownstairs last five completed: 1001 1000
6792025-01-17T21:18:03.974ZUpstairs last five completed: 1001 1000
6802025-01-17T21:18:03.974ZCLIENT: Up:0 ds:0 act:6
6812025-01-17T21:18:03.974ZCLIENT: All crucible jobs finished, exiting program
6822025-01-17T21:18:03.974Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:18:03.470334484Z","hostname":"w-01JHV1KGN2QVJYZ64NNY2VWD61","pid":1249}
6832025-01-17T21:18:03.978ZRepair part 2 passed
6842025-01-17T21:18:03.980Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
6852025-01-17T21:18:03.995ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6862025-01-17T21:18:03.995Z 0 000-009 32 32 32 3999 3999 3999 F F F
6872025-01-17T21:18:03.995Z 1 010-019 32 32 32 3999 3999 3999 F F F
6882025-01-17T21:18:03.995Z 2 020-029 32 32 32 3999 3999 3999 F F F
6892025-01-17T21:18:03.995Z 3 030-039 32 32 32 3999 3999 3999 F F F
6902025-01-17T21:18:03.995Z 4 040-049 32 32 32 3999 3999 3999 F F F
6912025-01-17T21:18:04.003ZMax gen: 32, Max flush: 3999
6922025-01-17T21:18:04.003Zdump test passed
6932025-01-17T21:18:04.003Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
6942025-01-17T21:18:04.015Z Extent 1
6952025-01-17T21:18:04.022ZGEN 32 32 32
6962025-01-17T21:18:04.022ZFLUSH_ID 3999 3999 3999
6972025-01-17T21:18:04.022ZDIRTY
6982025-01-17T21:18:04.022Z
6992025-01-17T21:18:04.022ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
7002025-01-17T21:18:04.022Z010 A A A A A A
7012025-01-17T21:18:04.022Z011 A A A A A A
7022025-01-17T21:18:04.022Z012 A A A A A A
7032025-01-17T21:18:04.026Z013 A A A A A A
7042025-01-17T21:18:04.026Z014 A A A A A A
7052025-01-17T21:18:04.026Z015 A A A A A A
7062025-01-17T21:18:04.026Z016 A A A A A A
7072025-01-17T21:18:04.026Z017 A A A A A A
7082025-01-17T21:18:04.029Z018 A A A A A A
7092025-01-17T21:18:04.029Z019 A A A A A A
7102025-01-17T21:18:04.036Zdump extent test passed
7112025-01-17T21:18:04.036Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
7122025-01-17T21:18:04.048ZExtent 2 Block in extent 0 Actual block 20
7132025-01-17T21:18:04.048Z
7142025-01-17T21:18:04.052Z DATA SHA256 VER
7152025-01-17T21:18:04.053Z------ ---------------------------------------------------------------- ---
7162025-01-17T21:18:04.053Z 0 b66158a613d4ced17f08e9c399e667aeef85e26912b11a5caad01c197b1a88ff A
7172025-01-17T21:18:04.053Z 1 b66158a613d4ced17f08e9c399e667aeef85e26912b11a5caad01c197b1a88ff A
7182025-01-17T21:18:04.053Z 2 b66158a613d4ced17f08e9c399e667aeef85e26912b11a5caad01c197b1a88ff A
7192025-01-17T21:18:04.053Z
7202025-01-17T21:18:04.053ZNONCES 0 1 2 DIFF
7212025-01-17T21:18:04.053Z------ ------------------------ ------------------------ ------------------------ -----
7222025-01-17T21:18:04.053Z 0 c804d0c32660f67e3a3fa585 c804d0c32660f67e3a3fa585 c804d0c32660f67e3a3fa585
7232025-01-17T21:18:04.053Z
7242025-01-17T21:18:04.053Z TAGS 0 1 2 DIFF
7252025-01-17T21:18:04.053Z------ -------------------------------- -------------------------------- -------------------------------- -----
7262025-01-17T21:18:04.053Z 0 a57668c92a22f7ad6070810588436b96 a57668c92a22f7ad6070810588436b96 a57668c92a22f7ad6070810588436b96
7272025-01-17T21:18:04.053Z
7282025-01-17T21:18:04.053ZHASHES 0 1 2 DIFF
7292025-01-17T21:18:04.053Z------ ---------------- ---------------- ---------------- -----
7302025-01-17T21:18:04.053Z
7312025-01-17T21:18:04.056Zdump block test passed
7322025-01-17T21:18:04.056ZInitial upstairs tests have completed, stopping all downstairs
7332025-01-17T21:18:09.120ZCreating 7 larger downstairs regions
7342025-01-17T21:18:09.299ZStarting 7 downstairs
7352025-01-17T21:18:14.309Zdsc restarted at PID: 1265
7362025-01-17T21:18:14.693ZNow do the replace-reconcile test
7372025-01-17T21:18:14.748ZUsing 8870 for the replacement port
7382025-01-17T21:20:52.101ZCompleted test: replace-reconcile
7392025-01-17T21:20:52.101ZNow do the replace-before-active test
7402025-01-17T21:24:25.488ZCompleted test: replace-before-active
7412025-01-17T21:24:25.489ZAll tests have completed, stopping all downstairs
7422025-01-17T21:24:25.548Z
7432025-01-17T21:24:25.548ZAll Tests have passed
7442025-01-17T21:24:25.552Z13:20 Test duration
7452025-01-17T21:24:25.552Z
7462025-01-17T21:24:25.553Zreal 13:19.733128226
7472025-01-17T21:24:25.553Zuser 18:20.234997452
7482025-01-17T21:24:25.553Zsys 24.300236872
7492025-01-17T21:24:25.553Ztrap 0.518603905
7502025-01-17T21:24:25.553Ztflt 0.040460771
7512025-01-17T21:24:25.553Zdflt 0.028742154
7522025-01-17T21:24:25.553Zkflt 0.001694934
7532025-01-17T21:24:25.553Zlock 25:24:28.682872517
7542025-01-17T21:24:25.553Zslp 1:13:59.283381904
7552025-01-17T21:24:25.553Zlat 2:00.959062126
7562025-01-17T21:24:25.553Zstop 0.199577105
7572025-01-17T21:24:25.553Z+ echo 'test-up-2region-encrypted ends'
7582025-01-17T21:24:25.553Ztest-up-2region-encrypted ends
7592025-01-17T21:24:30.555Zprocess exited: duration 808199 ms, exit code 0
7602025-01-17T21:24:30.555Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
7612025-01-17T21:25:30.610Zexec warning: : stdout descriptor may be held open by a background process; giving up!
7622025-01-17T21:25:30.610Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
7632025-01-17T21:25:30.621Zfound 17 output files
7642025-01-17T21:25:30.621Zuploading: /tmp/test_up-build/dsc-out.txt (48896 bytes)
7652025-01-17T21:25:31.633Zuploaded: /tmp/test_up-build/dsc-out.txt
7662025-01-17T21:25:31.633Zuploading: /tmp/test_up-build/test_up_out.txt (10409781 bytes)
7672025-01-17T21:25:32.809Zuploaded: /tmp/test_up-build/test_up_out.txt
7682025-01-17T21:25:32.809Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (147245 bytes)
7692025-01-17T21:25:33.821Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
7702025-01-17T21:25:33.821Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (144645 bytes)
7712025-01-17T21:25:34.833Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
7722025-01-17T21:25:34.836Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (251021 bytes)
7732025-01-17T21:25:35.847Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
7742025-01-17T21:25:35.847Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (139433 bytes)
7752025-01-17T21:25:36.859Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
7762025-01-17T21:25:36.862Zuploading: /tmp/test_up-build/dsc/downstairs-8850.txt (2493 bytes)
7772025-01-17T21:25:37.867Zuploaded: /tmp/test_up-build/dsc/downstairs-8850.txt
7782025-01-17T21:25:37.867Zuploading: /tmp/test_up-build/dsc/downstairs-8860.txt (33790 bytes)
7792025-01-17T21:25:38.877Zuploaded: /tmp/test_up-build/dsc/downstairs-8860.txt
7802025-01-17T21:25:38.877Zuploading: /tmp/test_up-build/dsc/downstairs-8870.txt (735763 bytes)
7812025-01-17T21:25:39.901Zuploaded: /tmp/test_up-build/dsc/downstairs-8870.txt
7822025-01-17T21:25:39.904Zuploading: /tmp/debug/df.txt (1270 bytes)
7832025-01-17T21:25:40.909Zuploaded: /tmp/debug/df.txt
7842025-01-17T21:25:40.909Zuploading: /tmp/debug/dtrace.txt (1002878 bytes)
7852025-01-17T21:25:41.935Zuploaded: /tmp/debug/dtrace.txt
7862025-01-17T21:25:41.938Zuploading: /tmp/debug/iostat.txt (259559 bytes)
7872025-01-17T21:25:41.947Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 259559 -> 263159
7882025-01-17T21:25:42.953Zuploaded: /tmp/debug/iostat.txt
7892025-01-17T21:25:42.953Zuploading: /tmp/debug/mpstat.txt (630600 bytes)
7902025-01-17T21:25:42.969Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 630600 -> 640077
7912025-01-17T21:25:43.976Zuploaded: /tmp/debug/mpstat.txt
7922025-01-17T21:25:43.978Zuploading: /tmp/debug/paging.txt (113221 bytes)
7932025-01-17T21:25:43.982Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 113221 -> 114989
7942025-01-17T21:25:44.988Zuploaded: /tmp/debug/paging.txt
7952025-01-17T21:25:44.991Zuploading: /tmp/debug/prstat.txt (1223939 bytes)
7962025-01-17T21:25:45.014Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 1223939 -> 1242016
7972025-01-17T21:25:46.020Zuploaded: /tmp/debug/prstat.txt
7982025-01-17T21:25:46.020Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
7992025-01-17T21:25:47.028Zuploaded: /tmp/debug/psrinfo.txt
8002025-01-17T21:25:47.028Zuploading: /tmp/debug/upstairs-info.txt (189478 bytes)
8012025-01-17T21:25:48.041Zuploaded: /tmp/debug/upstairs-info.txt