01JH9YKAYDRJNGA611ZRG1QBGS: test-up-2region-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01JH9YKR7TTE497A0XY6TPK3Y1

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12025-01-11T06:03:58.883Zjob dependencies complete; ready to run (waiting for 13 m 34 s)
22025-01-11T06:04:56.535Zjob assigned to worker 01JH9ZCR5JGH8B3VERD66A2P79 [factory aws, i-0c68e8afb19b47696] (queued for 57 s)
32025-01-11T06:05:05.037Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42025-01-11T06:05:07.032Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52025-01-11T06:05:07.032Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62025-01-11T06:05:08.120Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72025-01-11T06:05:08.120Zdownloading input: /input/build/work/bins/crutest.gz
82025-01-11T06:05:09.273Zdownloaded input: /input/build/work/bins/crutest.gz
92025-01-11T06:05:09.273Zdownloading input: /input/build/work/bins/dsc.gz
102025-01-11T06:05:09.961Zdownloaded input: /input/build/work/bins/dsc.gz
112025-01-11T06:05:09.963Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122025-01-11T06:05:09.977Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132025-01-11T06:05:09.977Zdownloading input: /input/build/work/scripts/test_ds.sh
142025-01-11T06:05:09.997Zdownloaded input: /input/build/work/scripts/test_ds.sh
152025-01-11T06:05:09.997Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162025-01-11T06:05:10.017Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172025-01-11T06:05:10.017Zdownloading input: /input/build/work/scripts/test_repair.sh
182025-01-11T06:05:10.033Zdownloaded input: /input/build/work/scripts/test_repair.sh
192025-01-11T06:05:10.033Zdownloading input: /input/build/work/scripts/test_replay.sh
202025-01-11T06:05:10.051Zdownloaded input: /input/build/work/scripts/test_replay.sh
212025-01-11T06:05:10.051Zdownloading input: /input/build/work/scripts/test_up.sh
222025-01-11T06:05:10.073Zdownloaded input: /input/build/work/scripts/test_up.sh
232025-01-11T06:05:10.073Zdownloading input: /input/build/work/scripts/upstairs_info.d
242025-01-11T06:05:10.092Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252025-01-11T06:05:10.095Zdownloading input: /input/build/tmp/cargo-test-out.log
262025-01-11T06:05:10.811Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272025-01-11T06:05:10.811Zstarting task 0: "setup"
282025-01-11T06:05:10.821Z++ uname -s
292025-01-11T06:05:10.824Z+ kern=SunOS
302025-01-11T06:05:10.824Z+ case "$kern" in
312025-01-11T06:05:10.824Z+ groupadd -g 12345 build
322025-01-11T06:05:10.827Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332025-01-11T06:05:12.835Z+ zfs create -o mountpoint=/work rpool/work
342025-01-11T06:05:13.229Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352025-01-11T06:05:13.236Z+ home_fs=zfs
362025-01-11T06:05:13.237Z+ [[ zfs == autofs ]]
372025-01-11T06:05:13.237Z+ mkdir -p /home/build
382025-01-11T06:05:13.237Z+ chown build:build /home/build /work
392025-01-11T06:05:14.243Z+ chmod 0700 /home/build /work
402025-01-11T06:05:14.250Zprocess exited: duration 3424 ms, exit code 0
 
412025-01-11T06:05:14.290Zstarting task 1: "authentication"
422025-01-11T06:05:14.325Zprocess exited: duration 31 ms, exit code 0
 
432025-01-11T06:05:14.398Zstarting task 2: "build"
442025-01-11T06:05:14.429Z+ banner cores
452025-01-11T06:05:14.429Z
462025-01-11T06:05:14.429Z #### #### ##### ###### ####
472025-01-11T06:05:14.429Z # # # # # # # #
482025-01-11T06:05:14.429Z # # # # # ##### ####
492025-01-11T06:05:14.429Z # # # ##### # #
502025-01-11T06:05:14.429Z # # # # # # # # #
512025-01-11T06:05:14.429Z #### #### # # ###### ####
522025-01-11T06:05:14.430Z
532025-01-11T06:05:14.430Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542025-01-11T06:05:14.430Z+ banner unpack
552025-01-11T06:05:14.430Z
562025-01-11T06:05:14.430Z # # # # ##### ## #### # #
572025-01-11T06:05:14.430Z # # ## # # # # # # # # #
582025-01-11T06:05:14.430Z # # # # # # # # # # ####
592025-01-11T06:05:14.430Z # # # # # ##### ###### # # #
602025-01-11T06:05:14.430Z # # # ## # # # # # # #
612025-01-11T06:05:14.430Z #### # # # # # #### # #
622025-01-11T06:05:14.430Z
632025-01-11T06:05:14.430Z+ mkdir -p /var/tmp/bins
642025-01-11T06:05:14.468Z+ for t in "$input/bins/"*.gz
652025-01-11T06:05:14.468Z++ basename /input/build/work/bins/crucible-downstairs.gz
662025-01-11T06:05:14.468Z+ b=crucible-downstairs.gz
672025-01-11T06:05:14.469Z+ b=crucible-downstairs
682025-01-11T06:05:14.470Z+ gunzip
692025-01-11T06:05:16.676Z+ chmod +x /var/tmp/bins/crucible-downstairs
702025-01-11T06:05:16.681Z+ for t in "$input/bins/"*.gz
712025-01-11T06:05:16.681Z++ basename /input/build/work/bins/crucible-hammer.gz
722025-01-11T06:05:16.681Z+ b=crucible-hammer.gz
732025-01-11T06:05:16.681Z+ b=crucible-hammer
742025-01-11T06:05:16.681Z+ gunzip
752025-01-11T06:05:18.906Z+ chmod +x /var/tmp/bins/crucible-hammer
762025-01-11T06:05:19.014Z+ for t in "$input/bins/"*.gz
772025-01-11T06:05:19.014Z++ basename /input/build/work/bins/crutest.gz
782025-01-11T06:05:19.014Z+ b=crutest.gz
792025-01-11T06:05:19.014Z+ b=crutest
802025-01-11T06:05:19.014Z+ gunzip
812025-01-11T06:05:21.209Z+ chmod +x /var/tmp/bins/crutest
822025-01-11T06:05:21.299Z+ for t in "$input/bins/"*.gz
832025-01-11T06:05:21.299Z++ basename /input/build/work/bins/dsc.gz
842025-01-11T06:05:21.299Z+ b=dsc.gz
852025-01-11T06:05:21.300Z+ b=dsc
862025-01-11T06:05:21.301Z+ gunzip
872025-01-11T06:05:22.628Z+ chmod +x /var/tmp/bins/dsc
882025-01-11T06:05:22.692Z+ export BINDIR=/var/tmp/bins
892025-01-11T06:05:22.692Z+ BINDIR=/var/tmp/bins
902025-01-11T06:05:22.692Z+ jobpid=1139
912025-01-11T06:05:22.692Z+ echo 'Setup debug logging'
922025-01-11T06:05:22.692ZSetup debug logging
932025-01-11T06:05:22.692Z+ mkdir /tmp/debug
942025-01-11T06:05:22.692Z+ sleep 3600
952025-01-11T06:05:22.692Z+ psrinfo -v
962025-01-11T06:05:22.692Z+ df -h
972025-01-11T06:05:22.692Z+ prstat -d d -mLc 1
982025-01-11T06:05:22.692Z+ iostat -T d -xn 1
992025-01-11T06:05:22.692Z+ mpstat -T d 1
1002025-01-11T06:05:22.692Z+ vmstat -T d -p 1
1012025-01-11T06:05:22.692Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1022025-01-11T06:05:22.692Z+ banner test_up_2r_encrypted
1032025-01-11T06:05:22.692Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1042025-01-11T06:05:22.693Z #####
1052025-01-11T06:05:22.693Z ##### ###### #### ##### # # ##### # # #####
1062025-01-11T06:05:22.693Z # # # # # # # # # # #
1072025-01-11T06:05:22.693Z # ##### #### # # # # # ##### # #
1082025-01-11T06:05:22.693Z # # # # # # ##### # #####
1092025-01-11T06:05:22.693Z # # # # # # # # # # #
1102025-01-11T06:05:22.694Z # ###### #### # ####### #### # ####### ####### # #
1112025-01-11T06:05:22.694Z
1122025-01-11T06:05:22.694Z+ ptime -m bash /input/build/work/scripts/test_up.sh -r 2 -N encrypted
1132025-01-11T06:05:22.694Z/input/build/work
1142025-01-11T06:05:22.775ZUsing 2 region sets
1152025-01-11T06:05:22.775ZTurn off color for downstairs dump
1162025-01-11T06:05:22.775ZUpstairs using key: lvIktV82mWuQq+slrAXkifBSQFAfPdeswXGaJt/qlf8=
1172025-01-11T06:05:22.775Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1182025-01-11T06:05:22.776ZCreating 6 downstairs regions
1192025-01-11T06:05:23.091ZStarting 6 downstairs
1202025-01-11T06:05:23.137Zdsc started at PID: 1187
1212025-01-11T06:05:28.431ZDisable automatic restart on all downstairs
1222025-01-11T06:05:28.491Z
1232025-01-11T06:05:28.492ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1242025-01-11T06:05:28.494ZRunning test: span
1252025-01-11T06:05:32.702ZCompleted test: span
1262025-01-11T06:05:37.719ZRunning test: big
1272025-01-11T06:05:42.347ZCompleted test: big
1282025-01-11T06:05:47.360ZRunning test: dep
1292025-01-11T06:06:20.495ZCompleted test: dep
1302025-01-11T06:06:25.502ZRunning test: balloon
1312025-01-11T06:10:37.637ZCompleted test: balloon
1322025-01-11T06:10:42.644ZRunning test: deactivate
1332025-01-11T06:11:37.402ZCompleted test: deactivate
1342025-01-11T06:11:42.409ZRunning hammer
1352025-01-11T06:12:00.391ZRun repair tests
1362025-01-11T06:12:00.394Z/var/tmp/bins/crutest fill -g 31 -q --key lvIktV82mWuQq+slrAXkifBSQFAfPdeswXGaJt/qlf8= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1372025-01-11T06:12:00.454Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.581209844Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"task":"crutest"}
1382025-01-11T06:12:00.456Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.58175713Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"task":"crutest"}
1392025-01-11T06:12:00.456Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.583423689Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"task":"crutest"}
1402025-01-11T06:12:00.456Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.583653922Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230}
1412025-01-11T06:12:00.456Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.583696682Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230}
1422025-01-11T06:12:00.456Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.583718852Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230}
1432025-01-11T06:12:00.456Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.583734063Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230}
1442025-01-11T06:12:00.456Z{"msg":"Crucible 4af9c30c-21f1-4ea1-9a5f-46dba2a7ee28 has session id: 0688273b-e226-4bdd-8331-08e84954b117","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.584277529Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1452025-01-11T06:12:00.456Z{"msg":"Upstairs opts: Upstairs UUID: 4af9c30c-21f1-4ea1-9a5f-46dba2a7ee28, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.584324819Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1462025-01-11T06:12:00.456Z{"msg":"Crucible stats registered with UUID: 4af9c30c-21f1-4ea1-9a5f-46dba2a7ee28","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.584341259Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1472025-01-11T06:12:00.493Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.620960081Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"task":"crutest"}
1482025-01-11T06:12:00.496Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.621075012Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230}
1492025-01-11T06:12:00.496Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.621110332Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230}
1502025-01-11T06:12:00.496Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.621136923Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230}
1512025-01-11T06:12:00.496Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.621155053Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230}
1522025-01-11T06:12:00.496Z{"msg":"Crucible 4af9c30c-21f1-4ea1-9a5f-46dba2a7ee28 has session id: da8bf09b-1d60-4d95-9cbc-d91291af9062","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.621613638Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
1532025-01-11T06:12:00.496Z{"msg":"Upstairs opts: Upstairs UUID: 4af9c30c-21f1-4ea1-9a5f-46dba2a7ee28, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.621665139Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
1542025-01-11T06:12:00.496Z{"msg":"Crucible stats registered with UUID: 4af9c30c-21f1-4ea1-9a5f-46dba2a7ee28","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.621694379Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
1552025-01-11T06:12:00.529Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.657060996Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"task":"crutest"}
1562025-01-11T06:12:00.529Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.657136867Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230}
1572025-01-11T06:12:00.529Z{"msg":"4af9c30c-21f1-4ea1-9a5f-46dba2a7ee28 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.657166457Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1582025-01-11T06:12:00.532Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.657209678Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"0","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1592025-01-11T06:12:00.532Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.657280089Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"0","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1602025-01-11T06:12:00.532Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.657340089Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"1","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1612025-01-11T06:12:00.532Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.6574016Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"1","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1622025-01-11T06:12:00.532Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.65742502Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"2","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1632025-01-11T06:12:00.532Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.657464831Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"2","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1642025-01-11T06:12:00.532Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.657496221Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1652025-01-11T06:12:00.532Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.657746944Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1662025-01-11T06:12:00.532Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.657810775Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1672025-01-11T06:12:00.532Z{"msg":"ds_connection connected from Ok(127.0.0.1:34566)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.657844665Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1682025-01-11T06:12:00.532Z{"msg":"ds_connection connected from Ok(127.0.0.1:55039)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.657873495Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1692025-01-11T06:12:00.532Z{"msg":"ds_connection connected from Ok(127.0.0.1:53415)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.657913926Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1702025-01-11T06:12:00.532Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.658420531Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"0","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1712025-01-11T06:12:00.532Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.658448622Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"0","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1722025-01-11T06:12:00.532Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.658462522Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"1","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1732025-01-11T06:12:00.532Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.658491212Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"1","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1742025-01-11T06:12:00.532Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.658505132Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"2","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1752025-01-11T06:12:00.532Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.658606524Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"2","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1762025-01-11T06:12:00.532Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.659042269Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"2","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1772025-01-11T06:12:00.532Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.659079989Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"1","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1782025-01-11T06:12:00.532Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.65918782Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"0","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1792025-01-11T06:12:00.533Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.659417953Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"2","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1802025-01-11T06:12:00.533Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.659477763Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"2","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1812025-01-11T06:12:00.533Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.659510074Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1822025-01-11T06:12:00.533Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.659530594Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"1","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1832025-01-11T06:12:00.533Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.659570784Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"1","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1842025-01-11T06:12:00.533Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.659609715Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1852025-01-11T06:12:00.533Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.659648205Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"0","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1862025-01-11T06:12:00.533Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.659685166Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"0","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1872025-01-11T06:12:00.533Z{"msg":"[0]R flush_numbers: [3996, 3996, 3996, 3996, 3996]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.659706136Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"downstairs","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1882025-01-11T06:12:00.537Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.659784167Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"downstairs","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1892025-01-11T06:12:00.537Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.659830457Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"downstairs","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1902025-01-11T06:12:00.537Z{"msg":"[1]R flush_numbers: [3996, 3996, 3996, 3996, 3996]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.659862668Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"downstairs","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1912025-01-11T06:12:00.537Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.659892138Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"downstairs","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1922025-01-11T06:12:00.537Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.66006961Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"downstairs","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1932025-01-11T06:12:00.537Z{"msg":"[2]R flush_numbers: [3996, 3996, 3996, 3996, 3996]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.660126101Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"downstairs","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1942025-01-11T06:12:00.537Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.660234182Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"downstairs","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1952025-01-11T06:12:00.537Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.660279502Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"downstairs","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1962025-01-11T06:12:00.538Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.660341863Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"downstairs","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1972025-01-11T06:12:00.538Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.660373503Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"downstairs","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1982025-01-11T06:12:00.538Z{"msg":"Next flush: 3997","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.660396344Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"downstairs","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
1992025-01-11T06:12:00.538Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.660410654Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"downstairs","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
2002025-01-11T06:12:00.538Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.660435004Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
2012025-01-11T06:12:00.538Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.660448674Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"0","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
2022025-01-11T06:12:00.538Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.660464744Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"1","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
2032025-01-11T06:12:00.538Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.660495315Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"2","":"downstairs","session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
2042025-01-11T06:12:00.538Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.660520315Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
2052025-01-11T06:12:00.538Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.660554375Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
2062025-01-11T06:12:00.538Z{"msg":"4af9c30c-21f1-4ea1-9a5f-46dba2a7ee28 is now active with session: 0688273b-e226-4bdd-8331-08e84954b117","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.660608046Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
2072025-01-11T06:12:00.538Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.660679237Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"session_id":"0688273b-e226-4bdd-8331-08e84954b117"}
2082025-01-11T06:12:00.538Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.660709527Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230}
2092025-01-11T06:12:00.538Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.660746868Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230}
2102025-01-11T06:12:00.538Z{"msg":"4af9c30c-21f1-4ea1-9a5f-46dba2a7ee28 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.660772028Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2112025-01-11T06:12:00.538Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.660793978Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"0","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2122025-01-11T06:12:00.538Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.660819628Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"0","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2132025-01-11T06:12:00.538Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.660848639Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"1","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2142025-01-11T06:12:00.538Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.660876799Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"1","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2152025-01-11T06:12:00.538Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.660908269Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"2","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2162025-01-11T06:12:00.538Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.660923119Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"2","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2172025-01-11T06:12:00.538Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.66094351Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2182025-01-11T06:12:00.538Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.66095816Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2192025-01-11T06:12:00.538Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.66097931Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2202025-01-11T06:12:00.538Z{"msg":"ds_connection connected from Ok(127.0.0.1:52102)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.66099327Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2212025-01-11T06:12:00.538Z{"msg":"ds_connection connected from Ok(127.0.0.1:64525)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.661035111Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2222025-01-11T06:12:00.538Z{"msg":"ds_connection connected from Ok(127.0.0.1:37313)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.661067491Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2232025-01-11T06:12:00.538Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.661112212Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"2","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2242025-01-11T06:12:00.538Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.661139902Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"2","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2252025-01-11T06:12:00.538Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.661175512Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"1","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2262025-01-11T06:12:00.538Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.661217553Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"1","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2272025-01-11T06:12:00.538Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.661247173Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"0","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2282025-01-11T06:12:00.538Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.661285334Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"0","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2292025-01-11T06:12:00.538Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.661316664Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"1","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2302025-01-11T06:12:00.538Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.661359064Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"2","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2312025-01-11T06:12:00.538Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.661400335Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"0","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2322025-01-11T06:12:00.538Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.661438255Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"1","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2332025-01-11T06:12:00.539Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.661478006Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"1","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2342025-01-11T06:12:00.539ZWait for a query_work_queue command to finish before sending IO
2352025-01-11T06:12:00.539Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.661522986Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2362025-01-11T06:12:00.539Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.661557177Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"2","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2372025-01-11T06:12:00.539Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.661592697Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"2","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2382025-01-11T06:12:00.539Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.661624347Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2392025-01-11T06:12:00.539Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.661652398Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"0","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2402025-01-11T06:12:00.539Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.661679018Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"0","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2412025-01-11T06:12:00.539Z{"msg":"[0]R flush_numbers: [5318, 5316, 5317, 5318, 5318]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.661717348Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"downstairs","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2422025-01-11T06:12:00.539Z{"msg":"[0]R generation: [14, 13, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.661746649Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"downstairs","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2432025-01-11T06:12:00.539Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.661786049Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"downstairs","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2442025-01-11T06:12:00.539Z{"msg":"[1]R flush_numbers: [5318, 5316, 5317, 5318, 5318]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.66182384Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"downstairs","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2452025-01-11T06:12:00.539Z{"msg":"[1]R generation: [14, 13, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.66185351Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"downstairs","":"Disk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
2462025-01-11T06:12:00.539Zdownstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2472025-01-11T06:12:00.539Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.661892661Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"downstairs","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2482025-01-11T06:12:00.539ZFill test
2492025-01-11T06:12:00.539Z{"msg":"[2]R flush_numbers: [5318, 5316, 5317, 5318, 5318]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.661921371Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"downstairs","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2502025-01-11T06:12:00.539Z{"msg":"[2]R generation: [14, 13, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.661935861Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"downstairs","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2512025-01-11T06:12:00.539Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.661956801Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"downstairs","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2522025-01-11T06:12:00.539Z{"msg":"Max found gen is 15","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.661970741Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"downstairs","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2532025-01-11T06:12:00.539Z{"msg":"Generation requested: 31 >= found:15","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.661991082Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"downstairs","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2542025-01-11T06:12:00.539Z{"msg":"Next flush: 5319","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.662005022Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"downstairs","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2552025-01-11T06:12:00.539Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.662025792Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"":"downstairs","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2562025-01-11T06:12:00.539Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.662040022Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2572025-01-11T06:12:00.539Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.662059152Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"0","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2582025-01-11T06:12:00.539Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.662072463Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"1","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2592025-01-11T06:12:00.539Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.662093593Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"client":"2","":"downstairs","session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2602025-01-11T06:12:00.539Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.662107563Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2612025-01-11T06:12:00.539Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.662130513Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2622025-01-11T06:12:00.539Z{"msg":"4af9c30c-21f1-4ea1-9a5f-46dba2a7ee28 is now active with session: da8bf09b-1d60-4d95-9cbc-d91291af9062","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.662164464Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2632025-01-11T06:12:00.539Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.662197554Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230,"session_id":"da8bf09b-1d60-4d95-9cbc-d91291af9062"}
2642025-01-11T06:12:00.539Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.662227384Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230}
2652025-01-11T06:12:00.576ZRead and Verify all blocks (0..100 range:false)
2662025-01-11T06:12:00.654ZWrote out file "/var/tmp/test_up-build/verify_file"
2672025-01-11T06:12:00.654ZCLIENT: Tests done. All submitted work has been ACK'd
2682025-01-11T06:12:00.657Z----------------------------------------------------------------
2692025-01-11T06:12:00.657Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2702025-01-11T06:12:00.657ZStates: Active Active Active
2712025-01-11T06:12:00.657ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2722025-01-11T06:12:00.657Z 1002 Acked Read 50 Done Done Done false
2732025-01-11T06:12:00.657Z STATES DS:0 DS:1 DS:2 TOTAL
2742025-01-11T06:12:00.657Z Sent 0 0 0 0
2752025-01-11T06:12:00.657Z Done 1 1 1 3
2762025-01-11T06:12:00.657Z Skipped 0 0 0 0
2772025-01-11T06:12:00.657Z Error 0 0 0 0
2782025-01-11T06:12:00.657ZLast Flush: 1001 1001 1001
2792025-01-11T06:12:00.657ZDownstairs last five completed: 1001 1000
2802025-01-11T06:12:00.657ZUpstairs last five completed: 1002 1001 1000
2812025-01-11T06:12:00.657Z----------------------------------------------------------------
2822025-01-11T06:12:00.657Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2832025-01-11T06:12:00.657ZStates: Active Active Active
2842025-01-11T06:12:00.657ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2852025-01-11T06:12:00.657Z 1002 Acked Read 50 Done Done Done false
2862025-01-11T06:12:00.657Z STATES DS:0 DS:1 DS:2 TOTAL
2872025-01-11T06:12:00.657Z Sent 0 0 0 0
2882025-01-11T06:12:00.657Z Done 1 1 1 3
2892025-01-11T06:12:00.657Z Skipped 0 0 0 0
2902025-01-11T06:12:00.657Z Error 0 0 0 0
2912025-01-11T06:12:00.657ZLast Flush: 1001 1001 1001
2922025-01-11T06:12:00.657ZDownstairs last five completed: 1001 1000
2932025-01-11T06:12:00.657ZUpstairs last five completed: 1002 1001 1000
2942025-01-11T06:12:00.657ZCLIENT: Up:0 ds:2 act:6
2952025-01-11T06:12:04.656Z----------------------------------------------------------------
2962025-01-11T06:12:04.656Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2972025-01-11T06:12:04.656ZDownstairs last five completed: 1003 1002 1001 1000
2982025-01-11T06:12:04.656ZUpstairs last five completed: 1003 1002 1001 1000
2992025-01-11T06:12:04.659Z----------------------------------------------------------------
3002025-01-11T06:12:04.659Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
3012025-01-11T06:12:04.659ZDownstairs last five completed: 1003 1002 1001 1000
3022025-01-11T06:12:04.659ZUpstairs last five completed: 1003 1002 1001 1000
3032025-01-11T06:12:04.659ZCLIENT: Up:0 ds:0 act:6
3042025-01-11T06:12:04.659ZCLIENT: All crucible jobs finished, exiting program
3052025-01-11T06:12:04.659Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:12:02.784079998Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1230}
3062025-01-11T06:12:04.668ZRepair setup passed
3072025-01-11T06:12:04.672ZCopy the region for /var/tmp/test_up-build/8830
3082025-01-11T06:12:04.674Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
3092025-01-11T06:12:04.677Z/var/tmp/bins/crutest fill -g 32 -q --key lvIktV82mWuQq+slrAXkifBSQFAfPdeswXGaJt/qlf8= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
3102025-01-11T06:12:04.738Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.863910975Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"task":"crutest"}
3112025-01-11T06:12:04.741Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.864493021Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"task":"crutest"}
3122025-01-11T06:12:04.741Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.866098799Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"task":"crutest"}
3132025-01-11T06:12:04.741Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.866305802Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236}
3142025-01-11T06:12:04.741Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.866384813Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236}
3152025-01-11T06:12:04.741Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.866428733Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236}
3162025-01-11T06:12:04.741Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.866447693Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236}
3172025-01-11T06:12:04.741Z{"msg":"Crucible c0cc6e7b-db16-4d98-820a-04d46a136156 has session id: 00019486-ecea-467c-b275-4bab91fc3810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.86701305Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3182025-01-11T06:12:04.741Z{"msg":"Upstairs opts: Upstairs UUID: c0cc6e7b-db16-4d98-820a-04d46a136156, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.86706894Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3192025-01-11T06:12:04.741Z{"msg":"Crucible stats registered with UUID: c0cc6e7b-db16-4d98-820a-04d46a136156","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.86709129Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3202025-01-11T06:12:04.775Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.902960233Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"task":"crutest"}
3212025-01-11T06:12:04.778Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.903083995Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236}
3222025-01-11T06:12:04.778Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.903113995Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236}
3232025-01-11T06:12:04.778Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.903127245Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236}
3242025-01-11T06:12:04.778Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.903141035Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236}
3252025-01-11T06:12:04.779Z{"msg":"Crucible c0cc6e7b-db16-4d98-820a-04d46a136156 has session id: 34cfd95b-c9c7-4492-ac82-dcdcc647bb36","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.903614091Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
3262025-01-11T06:12:04.779Z{"msg":"Upstairs opts: Upstairs UUID: c0cc6e7b-db16-4d98-820a-04d46a136156, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.903670281Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
3272025-01-11T06:12:04.779Z{"msg":"Crucible stats registered with UUID: c0cc6e7b-db16-4d98-820a-04d46a136156","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.903694701Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
3282025-01-11T06:12:04.810Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.938081188Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"task":"crutest"}
3292025-01-11T06:12:04.810Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.938159208Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236}
3302025-01-11T06:12:04.813Z{"msg":"c0cc6e7b-db16-4d98-820a-04d46a136156 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.938189629Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3312025-01-11T06:12:04.813Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.938236439Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"0","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3322025-01-11T06:12:04.813Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.93830006Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"0","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3332025-01-11T06:12:04.813Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.938362941Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"1","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3342025-01-11T06:12:04.813Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.938408621Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"1","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3352025-01-11T06:12:04.813Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.938452282Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"2","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3362025-01-11T06:12:04.813Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.938486302Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"2","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3372025-01-11T06:12:04.814Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.938510412Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"io task","client":"2","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3382025-01-11T06:12:04.814Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.938527123Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"io task","client":"0","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3392025-01-11T06:12:04.814Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.938565673Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"io task","client":"1","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3402025-01-11T06:12:04.814Z{"msg":"ds_connection connected from Ok(127.0.0.1:32866)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.938596684Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"io task","client":"2","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3412025-01-11T06:12:04.814Z{"msg":"ds_connection connected from Ok(127.0.0.1:37598)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.938638134Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"io task","client":"0","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3422025-01-11T06:12:04.814Z{"msg":"ds_connection connected from Ok(127.0.0.1:42561)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.938683094Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"io task","client":"1","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3432025-01-11T06:12:04.814Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.939462383Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"1","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3442025-01-11T06:12:04.814Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.939522694Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"1","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3452025-01-11T06:12:04.814Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.939584595Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"0","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3462025-01-11T06:12:04.814Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.939634035Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"0","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3472025-01-11T06:12:04.814Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.939652765Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"2","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3482025-01-11T06:12:04.814Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.939680956Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"2","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3492025-01-11T06:12:04.814Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.940165571Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"0","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3502025-01-11T06:12:04.814Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.940310763Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"1","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3512025-01-11T06:12:04.814Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.940343823Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"2","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3522025-01-11T06:12:04.814Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.940505235Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"0","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3532025-01-11T06:12:04.814Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.940542375Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"0","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3542025-01-11T06:12:04.814Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.940590586Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3552025-01-11T06:12:04.815Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.940666967Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"2","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3562025-01-11T06:12:04.815Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.940703317Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"2","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3572025-01-11T06:12:04.815Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.940718917Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3582025-01-11T06:12:04.815Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.940738557Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"1","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3592025-01-11T06:12:04.815Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.940752248Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"1","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3602025-01-11T06:12:04.815Z{"msg":"[0]R flush_numbers: [3997, 3997, 3997, 3997, 3997]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.940772658Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"downstairs","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3612025-01-11T06:12:04.815Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.940786808Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"downstairs","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3622025-01-11T06:12:04.815Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.940806708Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"downstairs","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3632025-01-11T06:12:04.815Z{"msg":"[1]R flush_numbers: [3997, 3997, 3997, 3997, 3997]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.940820098Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"downstairs","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3642025-01-11T06:12:04.815Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.940841649Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"downstairs","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3652025-01-11T06:12:04.815Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.940855089Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"downstairs","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3662025-01-11T06:12:04.815Z{"msg":"[2]R flush_numbers: [3997, 3997, 3997, 3997, 3997]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.940868699Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"downstairs","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3672025-01-11T06:12:04.815Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.940881689Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"downstairs","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3682025-01-11T06:12:04.815Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.940895009Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"downstairs","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3692025-01-11T06:12:04.815Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.940907789Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"downstairs","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3702025-01-11T06:12:04.815Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.94092563Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"downstairs","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3712025-01-11T06:12:04.815Z{"msg":"Next flush: 3998","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.94096104Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"downstairs","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3722025-01-11T06:12:04.815Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.94099706Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"downstairs","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3732025-01-11T06:12:04.815Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.941025801Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3742025-01-11T06:12:04.818Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.941056921Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"0","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3752025-01-11T06:12:04.818Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.941100241Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"1","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3762025-01-11T06:12:04.818Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.941129872Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"2","":"downstairs","session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3772025-01-11T06:12:04.818Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.941158572Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3782025-01-11T06:12:04.818Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.941328444Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3792025-01-11T06:12:04.819Z{"msg":"c0cc6e7b-db16-4d98-820a-04d46a136156 is now active with session: 00019486-ecea-467c-b275-4bab91fc3810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.941369095Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3802025-01-11T06:12:04.819Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.941443545Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"session_id":"00019486-ecea-467c-b275-4bab91fc3810"}
3812025-01-11T06:12:04.819Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.941494336Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236}
3822025-01-11T06:12:04.819Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.941521036Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236}
3832025-01-11T06:12:04.819Z{"msg":"c0cc6e7b-db16-4d98-820a-04d46a136156 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.941566167Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
3842025-01-11T06:12:04.819Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.941594317Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"0","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
3852025-01-11T06:12:04.819Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.941642938Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"0","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
3862025-01-11T06:12:04.819Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.941668848Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"1","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
3872025-01-11T06:12:04.819Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.941688348Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"1","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
3882025-01-11T06:12:04.819Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.941707378Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"2","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
3892025-01-11T06:12:04.819Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.941731709Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"2","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
3902025-01-11T06:12:04.819Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.941756209Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"io task","client":"2","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
3912025-01-11T06:12:04.819Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.941792569Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"io task","client":"0","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
3922025-01-11T06:12:04.819Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.94181806Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"io task","client":"1","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
3932025-01-11T06:12:04.819Z{"msg":"ds_connection connected from Ok(127.0.0.1:60343)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.94184187Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"io task","client":"2","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
3942025-01-11T06:12:04.819Z{"msg":"ds_connection connected from Ok(127.0.0.1:37840)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.94186662Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"io task","client":"0","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
3952025-01-11T06:12:04.819Z{"msg":"ds_connection connected from Ok(127.0.0.1:51773)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.94189241Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"io task","client":"1","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
3962025-01-11T06:12:04.819Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.941922121Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"1","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
3972025-01-11T06:12:04.819Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.941948641Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"1","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
3982025-01-11T06:12:04.819Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.941993351Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"0","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
3992025-01-11T06:12:04.819Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.942022042Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"0","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4002025-01-11T06:12:04.819Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.942064002Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"2","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4012025-01-11T06:12:04.819Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.942092453Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"2","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4022025-01-11T06:12:04.819Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.942241034Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"1","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4032025-01-11T06:12:04.819Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.942401516Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"0","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4042025-01-11T06:12:04.819Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.942443857Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"2","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4052025-01-11T06:12:04.819Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.942584718Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"1","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4062025-01-11T06:12:04.819Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.942625649Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"1","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4072025-01-11T06:12:04.820Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.942641119Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4082025-01-11T06:12:04.820Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.94274169Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"2","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4092025-01-11T06:12:04.820Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.94276748Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"2","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4102025-01-11T06:12:04.820Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.942781271Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4112025-01-11T06:12:04.820Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.942801341Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"0","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4122025-01-11T06:12:04.820Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.942821771Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"0","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4132025-01-11T06:12:04.820Z{"msg":"[0]R flush_numbers: [5319, 5319, 5319, 5319, 5319]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.942835281Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"downstairs","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4142025-01-11T06:12:04.820Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.942849161Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"downstairs","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4152025-01-11T06:12:04.820Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.942862061Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"downstairs","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4162025-01-11T06:12:04.820Z{"msg":"[1]R flush_numbers: [5319, 5319, 5319, 5319, 5319]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.942875081Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"downstairs","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4172025-01-11T06:12:04.820Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.942887952Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"downstairs","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4182025-01-11T06:12:04.820Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.942900792Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"downstairs","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4192025-01-11T06:12:04.820Z{"msg":"[2]R flush_numbers: [5319, 5319, 5319, 5319, 5319]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.942913662Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"downstairs","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4202025-01-11T06:12:04.820Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.942941112Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"downstairs","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4212025-01-11T06:12:04.820Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
4222025-01-11T06:12:04.820Z,"time":"2025-01-11T06:12:02.942966382Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"downstairs","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4232025-01-11T06:12:04.820Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.942994603Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"downstairs","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4242025-01-11T06:12:04.820Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.943008313Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"downstairs","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4252025-01-11T06:12:04.820Z{"msg":"Next flush: 5320","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.943020953Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"downstairs","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4262025-01-11T06:12:04.820Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.943034543Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"":"downstairs","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4272025-01-11T06:12:04.820Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.943047183Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4282025-01-11T06:12:04.820Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.943058934Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"0","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4292025-01-11T06:12:04.820Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.943080514Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"1","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4302025-01-11T06:12:04.820Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.943093894Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"client":"2","":"downstairs","session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4312025-01-11T06:12:04.820Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.943106684Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4322025-01-11T06:12:04.820Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.943125984Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4332025-01-11T06:12:04.820Z{"msg":"c0cc6e7b-db16-4d98-820a-04d46a136156 is now active with session: 34cfd95b-c9c7-4492-ac82-dcdcc647bb36","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.943138975Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4342025-01-11T06:12:04.821Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.943150555Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236,"session_id":"34cfd95b-c9c7-4492-ac82-dcdcc647bb36"}
4352025-01-11T06:12:04.821Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.943162405Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236}
4362025-01-11T06:12:04.821ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
4372025-01-11T06:12:04.821ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4382025-01-11T06:12:04.821ZFill test
4392025-01-11T06:12:04.857ZRead and Verify all blocks (0..100 range:false)
4402025-01-11T06:12:04.938ZWrote out file "/var/tmp/test_up-build/verify_file"
4412025-01-11T06:12:04.938ZCLIENT: Tests done. All submitted work has been ACK'd
4422025-01-11T06:12:04.941Z----------------------------------------------------------------
4432025-01-11T06:12:04.941Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4442025-01-11T06:12:04.941ZStates: Active Active Active
4452025-01-11T06:12:04.941ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4462025-01-11T06:12:04.941Z 1002 Acked Read 50 Done Done Done false
4472025-01-11T06:12:04.941Z STATES DS:0 DS:1 DS:2 TOTAL
4482025-01-11T06:12:04.941Z Sent 0 0 0 0
4492025-01-11T06:12:04.941Z Done 1 1 1 3
4502025-01-11T06:12:04.941Z Skipped 0 0 0 0
4512025-01-11T06:12:04.941Z Error 0 0 0 0
4522025-01-11T06:12:04.941ZLast Flush: 1001 1001 1001
4532025-01-11T06:12:04.941ZDownstairs last five completed: 1001 1000
4542025-01-11T06:12:04.941ZUpstairs last five completed: 1002 1001 1000
4552025-01-11T06:12:04.941Z----------------------------------------------------------------
4562025-01-11T06:12:04.941Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4572025-01-11T06:12:04.941ZStates: Active Active Active
4582025-01-11T06:12:04.941ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4592025-01-11T06:12:04.941Z 1002 Acked Read 50 Done Done Done false
4602025-01-11T06:12:04.941Z STATES DS:0 DS:1 DS:2 TOTAL
4612025-01-11T06:12:04.941Z Sent 0 0 0 0
4622025-01-11T06:12:04.941Z Done 1 1 1 3
4632025-01-11T06:12:04.941Z Skipped 0 0 0 0
4642025-01-11T06:12:04.941Z Error 0 0 0 0
4652025-01-11T06:12:04.941ZLast Flush: 1001 1001 1001
4662025-01-11T06:12:04.941ZDownstairs last five completed: 1001 1000
4672025-01-11T06:12:04.942ZUpstairs last five completed: 1002 1001 1000
4682025-01-11T06:12:04.942ZCLIENT: Up:0 ds:2 act:6
4692025-01-11T06:12:08.939Z----------------------------------------------------------------
4702025-01-11T06:12:08.940Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4712025-01-11T06:12:08.940ZDownstairs last five completed: 1003 1002 1001 1000
4722025-01-11T06:12:08.940ZUpstairs last five completed: 1003 1002 1001 1000
4732025-01-11T06:12:08.942Z----------------------------------------------------------------
4742025-01-11T06:12:08.942Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4752025-01-11T06:12:08.942ZDownstairs last five completed: 1003 1002 1001 1000
4762025-01-11T06:12:08.942ZUpstairs last five completed: 1003 1002 1001 1000
4772025-01-11T06:12:08.942ZCLIENT: Up:0 ds:0 act:6
4782025-01-11T06:12:08.942ZCLIENT: All crucible jobs finished, exiting program
4792025-01-11T06:12:08.942Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:12:07.067713989Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1236}
4802025-01-11T06:12:08.950ZRepair part 1 passed
4812025-01-11T06:12:08.953Z
4822025-01-11T06:12:08.953ZKill the current downstairs
4832025-01-11T06:12:09.129ZDownstairs 2 stopped
4842025-01-11T06:12:09.129Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
4852025-01-11T06:12:09.132ZNow put back the original so we have a mismatch
4862025-01-11T06:12:09.132Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
4872025-01-11T06:12:09.134ZRestart downstairs with old directory
4882025-01-11T06:12:09.192Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4892025-01-11T06:12:09.211ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4902025-01-11T06:12:09.213Z 0 000-009 32 32 31 3998 3998 3997 F F F <---
4912025-01-11T06:12:09.213Z 1 010-019 32 32 31 3998 3998 3997 F F F <---
4922025-01-11T06:12:09.213Z 2 020-029 32 32 31 3998 3998 3997 F F F <---
4932025-01-11T06:12:09.214Z 3 030-039 32 32 31 3998 3998 3997 F F F <---
4942025-01-11T06:12:09.214Z 4 040-049 32 32 31 3998 3998 3997 F F F <---
4952025-01-11T06:12:09.214ZMax gen: 32, Max flush: 3998
4962025-01-11T06:12:09.214ZError: Difference in extent metadata found!
4972025-01-11T06:12:09.216Zdump test found error as expected
4982025-01-11T06:12:09.216Z
4992025-01-11T06:12:09.216Z
5002025-01-11T06:12:09.216Z/var/tmp/bins/crutest verify --range -g 33 -q --key lvIktV82mWuQq+slrAXkifBSQFAfPdeswXGaJt/qlf8= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
5012025-01-11T06:12:09.276Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.403300187Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"task":"crutest"}
5022025-01-11T06:12:09.279Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.403888914Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"task":"crutest"}
5032025-01-11T06:12:09.279Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.405613573Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"task":"crutest"}
5042025-01-11T06:12:09.279Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.405830206Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244}
5052025-01-11T06:12:09.279Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.405891367Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244}
5062025-01-11T06:12:09.279Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.405914857Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244}
5072025-01-11T06:12:09.279Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.405930417Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244}
5082025-01-11T06:12:09.279Z{"msg":"Crucible 258d6810-2a67-4a2a-9048-1d11597b14df has session id: 1295666f-1480-4e3c-b741-32f7e320c6f6","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.406487743Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5092025-01-11T06:12:09.280Z{"msg":"Upstairs opts: Upstairs UUID: 258d6810-2a67-4a2a-9048-1d11597b14df, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.406551144Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5102025-01-11T06:12:09.280Z{"msg":"Crucible stats registered with UUID: 258d6810-2a67-4a2a-9048-1d11597b14df","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.406589475Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5112025-01-11T06:12:09.315Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.44266481Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"task":"crutest"}
5122025-01-11T06:12:09.315Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.442766191Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244}
5132025-01-11T06:12:09.318Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.442814771Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244}
5142025-01-11T06:12:09.318Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.442847162Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244}
5152025-01-11T06:12:09.318Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.442872572Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244}
5162025-01-11T06:12:09.318Z{"msg":"Crucible 258d6810-2a67-4a2a-9048-1d11597b14df has session id: 8a46df86-4105-47a5-bec1-811e292510a7","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.443260536Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
5172025-01-11T06:12:09.318Z{"msg":"Upstairs opts: Upstairs UUID: 258d6810-2a67-4a2a-9048-1d11597b14df, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.443336297Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
5182025-01-11T06:12:09.318Z{"msg":"Crucible stats registered with UUID: 258d6810-2a67-4a2a-9048-1d11597b14df","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.443361857Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
5192025-01-11T06:12:09.350Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.477886215Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"task":"crutest"}
5202025-01-11T06:12:09.350Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.477969976Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244}
5212025-01-11T06:12:09.350Z{"msg":"258d6810-2a67-4a2a-9048-1d11597b14df active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.477997466Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5222025-01-11T06:12:09.350Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.478025607Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"0","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5232025-01-11T06:12:09.350Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.478082357Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"0","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5242025-01-11T06:12:09.351Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.478114858Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"1","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5252025-01-11T06:12:09.353Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.478145808Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"1","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5262025-01-11T06:12:09.353Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.478175748Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"2","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5272025-01-11T06:12:09.353Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.478201439Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"2","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5282025-01-11T06:12:09.353Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.478227729Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5292025-01-11T06:12:09.354Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.478253889Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5302025-01-11T06:12:09.354Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.47831393Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5312025-01-11T06:12:09.354Z{"msg":"ds_connection connected from Ok(127.0.0.1:53453)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.47836111Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5322025-01-11T06:12:09.354Z{"msg":"ds_connection connected from Ok(127.0.0.1:35905)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.478395501Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5332025-01-11T06:12:09.354Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:12:07.478433511Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5342025-01-11T06:12:09.354Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:12:07.478467782Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5352025-01-11T06:12:09.354Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:12:07.478664274Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5362025-01-11T06:12:09.354Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.478770365Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5372025-01-11T06:12:09.354Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.479164059Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"1","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5382025-01-11T06:12:09.354Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.47920163Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"1","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5392025-01-11T06:12:09.354Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.47921771Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"0","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5402025-01-11T06:12:09.354Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.47924084Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"0","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5412025-01-11T06:12:09.354Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.479815597Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"0","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5422025-01-11T06:12:09.354Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.479969268Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"1","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5432025-01-11T06:12:09.354Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.480213031Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"0","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5442025-01-11T06:12:09.354Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.480255792Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"0","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5452025-01-11T06:12:09.354Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.480282242Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5462025-01-11T06:12:09.354Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.480303732Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"1","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5472025-01-11T06:12:09.354Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.480327423Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"1","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5482025-01-11T06:12:09.354Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.480358033Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5492025-01-11T06:12:19.353Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.480680547Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5502025-01-11T06:12:19.353Z{"msg":"ds_connection connected from Ok(127.0.0.1:54378)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.480789858Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5512025-01-11T06:12:19.357Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.481776579Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"2","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5522025-01-11T06:12:19.357Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.48182843Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"2","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5532025-01-11T06:12:19.357Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.482397466Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"2","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5542025-01-11T06:12:19.357Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.482813271Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"2","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5552025-01-11T06:12:19.357Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.482849231Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"2","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5562025-01-11T06:12:19.357Z{"msg":"[0]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.482868291Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5572025-01-11T06:12:19.357Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.482894942Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5582025-01-11T06:12:19.357Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.482912952Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5592025-01-11T06:12:19.357Z{"msg":"[1]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.482938492Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5602025-01-11T06:12:19.357Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.482964472Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5612025-01-11T06:12:19.357Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.482981152Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5622025-01-11T06:12:19.357Z{"msg":"[2]R flush_numbers: [3997, 3997, 3997, 3997, 3997]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.483013913Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5632025-01-11T06:12:19.357Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.483050023Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5642025-01-11T06:12:19.357Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.483077124Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5652025-01-11T06:12:19.357Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.483094644Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5662025-01-11T06:12:19.357Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.483119594Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5672025-01-11T06:12:19.357Z{"msg":"Next flush: 3999","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.483136924Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5682025-01-11T06:12:19.357Z{"msg":"Extent 0 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.483161465Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5692025-01-11T06:12:19.357Z{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.483179615Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5702025-01-11T06:12:19.358Z{"msg":"extent:0 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.483205615Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5712025-01-11T06:12:19.358Z{"msg":"extent:0 flush: 3998 3998 3997 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.483224355Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5722025-01-11T06:12:19.358Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.483250155Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5732025-01-11T06:12:19.358Z{"msg":"extent:0 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.483267866Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5742025-01-11T06:12:19.358Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.483293796Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5752025-01-11T06:12:19.358Z{"msg":"find dest for source 0 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.483311786Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5762025-01-11T06:12:19.358Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.483329607Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5772025-01-11T06:12:19.358Z{"msg":"Extent 1 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.483382997Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5782025-01-11T06:12:19.358Z{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.483433188Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5792025-01-11T06:12:19.358Z{"msg":"extent:1 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.483486038Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5802025-01-11T06:12:19.358Z{"msg":"extent:1 flush: 3998 3998 3997 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.483533879Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5812025-01-11T06:12:19.358Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.483577299Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5822025-01-11T06:12:19.358Z{"msg":"extent:1 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.483597719Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5832025-01-11T06:12:19.358Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.48363535Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5842025-01-11T06:12:19.358Z{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.48366824Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5852025-01-11T06:12:19.362Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.483696571Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5862025-01-11T06:12:19.362Z{"msg":"Extent 2 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.483741741Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5872025-01-11T06:12:19.362Z{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.483775012Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5882025-01-11T06:12:19.362Z{"msg":"extent:2 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.483820552Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5892025-01-11T06:12:19.362Z{"msg":"extent:2 flush: 3998 3998 3997 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.483865942Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5902025-01-11T06:12:19.362Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.483883253Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5912025-01-11T06:12:19.362Z{"msg":"extent:2 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.483904843Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5922025-01-11T06:12:19.362Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.483919893Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5932025-01-11T06:12:19.362Z{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.483940843Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5942025-01-11T06:12:19.362Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.483955114Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5952025-01-11T06:12:19.362Z{"msg":"Extent 3 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.483976254Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5962025-01-11T06:12:19.362Z{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.483991904Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5972025-01-11T06:12:19.362Z{"msg":"extent:3 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.484015484Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5982025-01-11T06:12:19.362Z{"msg":"extent:3 flush: 3998 3998 3997 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.484033904Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
5992025-01-11T06:12:19.362Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.484059445Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6002025-01-11T06:12:19.362Z{"msg":"extent:3 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.484074065Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6012025-01-11T06:12:19.362Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.484088225Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6022025-01-11T06:12:19.362Z{"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.484109375Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6032025-01-11T06:12:19.362Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.484123355Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6042025-01-11T06:12:19.362Z{"msg":"Extent 4 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.484137405Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6052025-01-11T06:12:19.362Z{"msg":"First source client ID for extent 4","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.484163306Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6062025-01-11T06:12:19.362Z{"msg":"extent:4 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.484177706Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6072025-01-11T06:12:19.362Z{"msg":"extent:4 flush: 3998 3998 3997 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.484191736Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6082025-01-11T06:12:19.362Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.484212566Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6092025-01-11T06:12:19.362Z{"msg":"extent:4 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.484226586Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6102025-01-11T06:12:19.363Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.484240447Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6112025-01-11T06:12:19.363Z{"msg":"find dest for source 0 for extent at index 4","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.484261087Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6122025-01-11T06:12:19.363Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.484274967Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6132025-01-11T06:12:19.363Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.484300187Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"0","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6142025-01-11T06:12:19.363Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.484321378Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"1","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6152025-01-11T06:12:19.363Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.484334878Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"2","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6162025-01-11T06:12:19.363Z{"msg":"Full repair list: {ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.484352928Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6172025-01-11T06:12:19.363Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 3999, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 3999, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 3999, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 3999, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 3999, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.484433899Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6182025-01-11T06:12:19.363Z{"msg":"starting reconciliation f8747872-d6ab-4102-91d1-d9ada8e11843: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.484449409Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6192025-01-11T06:12:19.363Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-11T06:12:17.484462569Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"reconcile":"f8747872-d6ab-4102-91d1-d9ada8e11843","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6202025-01-11T06:12:19.363Z{"msg":"no Nexus client from DNS, aborting start notification","v":0,"name":"crucible","level":50,"time":"2025-01-11T06:12:17.484485929Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"reconcile":"f8747872-d6ab-4102-91d1-d9ada8e11843","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6212025-01-11T06:12:19.363Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-11T06:12:17.484521Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"reconcile":"f8747872-d6ab-4102-91d1-d9ada8e11843","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6222025-01-11T06:12:19.363Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.4845427Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"2","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6232025-01-11T06:12:19.363Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-11T06:12:17.48456455Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"reconcile":"f8747872-d6ab-4102-91d1-d9ada8e11843","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6242025-01-11T06:12:19.366Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-11T06:12:17.491931853Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"reconcile":"f8747872-d6ab-4102-91d1-d9ada8e11843","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6252025-01-11T06:12:19.366Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-11T06:12:17.492369098Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"reconcile":"f8747872-d6ab-4102-91d1-d9ada8e11843","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6262025-01-11T06:12:19.366Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-11T06:12:17.492599091Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"reconcile":"f8747872-d6ab-4102-91d1-d9ada8e11843","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6272025-01-11T06:12:19.366Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.492906184Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"2","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6282025-01-11T06:12:19.366Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-11T06:12:17.492942264Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"reconcile":"f8747872-d6ab-4102-91d1-d9ada8e11843","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6292025-01-11T06:12:19.369Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-11T06:12:17.495478433Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"reconcile":"f8747872-d6ab-4102-91d1-d9ada8e11843","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6302025-01-11T06:12:19.369Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-11T06:12:17.495887548Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"reconcile":"f8747872-d6ab-4102-91d1-d9ada8e11843","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6312025-01-11T06:12:19.369Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-11T06:12:17.49612861Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"reconcile":"f8747872-d6ab-4102-91d1-d9ada8e11843","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6322025-01-11T06:12:19.369Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.496419954Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"2","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6332025-01-11T06:12:19.369Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-11T06:12:17.496458894Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"reconcile":"f8747872-d6ab-4102-91d1-d9ada8e11843","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6342025-01-11T06:12:19.371Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-11T06:12:17.499019453Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"reconcile":"f8747872-d6ab-4102-91d1-d9ada8e11843","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6352025-01-11T06:12:19.371Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-11T06:12:17.499396307Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"reconcile":"f8747872-d6ab-4102-91d1-d9ada8e11843","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6362025-01-11T06:12:19.374Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-11T06:12:17.49964256Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"reconcile":"f8747872-d6ab-4102-91d1-d9ada8e11843","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6372025-01-11T06:12:19.374Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.499965403Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"2","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6382025-01-11T06:12:19.374Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-11T06:12:17.500001954Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"reconcile":"f8747872-d6ab-4102-91d1-d9ada8e11843","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6392025-01-11T06:12:19.376Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-11T06:12:17.502651244Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"reconcile":"f8747872-d6ab-4102-91d1-d9ada8e11843","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6402025-01-11T06:12:19.376Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-11T06:12:17.503071178Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"reconcile":"f8747872-d6ab-4102-91d1-d9ada8e11843","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6412025-01-11T06:12:19.376Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-11T06:12:17.503322931Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"reconcile":"f8747872-d6ab-4102-91d1-d9ada8e11843","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6422025-01-11T06:12:19.376Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.503637255Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"2","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6432025-01-11T06:12:19.376Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-11T06:12:17.503672825Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"reconcile":"f8747872-d6ab-4102-91d1-d9ada8e11843","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6442025-01-11T06:12:19.379Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-11T06:12:17.506449966Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"reconcile":"f8747872-d6ab-4102-91d1-d9ada8e11843","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6452025-01-11T06:12:19.379Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.50678007Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6462025-01-11T06:12:19.379Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.50682666Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"0","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6472025-01-11T06:12:19.382Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.506854851Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"1","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6482025-01-11T06:12:19.382Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.506891341Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"2","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6492025-01-11T06:12:19.382Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.506935352Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6502025-01-11T06:12:19.382Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.506976812Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6512025-01-11T06:12:19.382Z{"msg":"258d6810-2a67-4a2a-9048-1d11597b14df is now active with session: 1295666f-1480-4e3c-b741-32f7e320c6f6","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.507013772Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6522025-01-11T06:12:19.382Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.507050373Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244}
6532025-01-11T06:12:19.382Z{"msg":"no Nexus client from DNS, aborting finish notification","v":0,"name":"crucible","level":50,"time":"2025-01-11T06:12:17.507075763Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"reconcile":"f8747872-d6ab-4102-91d1-d9ada8e11843","":"downstairs","":"downstairs","session_id":"1295666f-1480-4e3c-b741-32f7e320c6f6"}
6542025-01-11T06:12:19.382Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.507106714Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244}
6552025-01-11T06:12:19.382Z{"msg":"258d6810-2a67-4a2a-9048-1d11597b14df active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.507153544Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6562025-01-11T06:12:19.382Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.507185274Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"0","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6572025-01-11T06:12:19.382Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.507215985Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"0","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6582025-01-11T06:12:19.382Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.507241245Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"1","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6592025-01-11T06:12:19.382Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.507265965Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"1","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6602025-01-11T06:12:19.382Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.507290265Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"2","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6612025-01-11T06:12:19.382Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.507312776Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"2","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6622025-01-11T06:12:19.382Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.507335876Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6632025-01-11T06:12:19.382Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.507361306Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6642025-01-11T06:12:19.382Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.507384866Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6652025-01-11T06:12:19.382Z{"msg":"ds_connection connected from Ok(127.0.0.1:52319)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.507412817Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6662025-01-11T06:12:19.382Z{"msg":"ds_connection connected from Ok(127.0.0.1:50336)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.507440127Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6672025-01-11T06:12:19.383Z{"msg":"ds_connection connected from Ok(127.0.0.1:40629)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.507469558Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6682025-01-11T06:12:19.383Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.507610179Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"0","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6692025-01-11T06:12:19.383Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.5076594Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"0","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6702025-01-11T06:12:19.383Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.50769635Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"2","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6712025-01-11T06:12:19.383Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.50772105Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"2","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6722025-01-11T06:12:19.383Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.507744841Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"1","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6732025-01-11T06:12:19.383Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.507759431Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"1","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6742025-01-11T06:12:19.383Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.508185555Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"1","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6752025-01-11T06:12:19.383Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.508323037Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"0","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6762025-01-11T06:12:19.383Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.508360337Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"2","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6772025-01-11T06:12:19.383Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.508517389Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"1","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6782025-01-11T06:12:19.383Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.50854612Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"1","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6792025-01-11T06:12:19.383Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.50856116Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6802025-01-11T06:12:19.383Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.50862484Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"2","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6812025-01-11T06:12:19.383Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.508652341Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"2","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6822025-01-11T06:12:19.383Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.508666721Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6832025-01-11T06:12:19.383Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.508696981Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"0","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6842025-01-11T06:12:19.383Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.508719861Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"0","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6852025-01-11T06:12:19.383Z{"msg":"[0]R flush_numbers: [5320, 5320, 5320, 5320, 5320]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.508735132Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"downstairs","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6862025-01-11T06:12:19.383Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.508759342Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"downstairs","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6872025-01-11T06:12:19.383Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.508781032Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"downstairs","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6882025-01-11T06:12:19.383Z{"msg":"[1]R flush_numbers: [5320, 5320, 5320, 5320, 5320]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.508800193Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO
6892025-01-11T06:12:19.383Zdownstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6902025-01-11T06:12:19.383Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.508837363Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"downstairs","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6912025-01-11T06:12:19.383Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.508863833Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"downstairs","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6922025-01-11T06:12:19.383Z{"msg":"[2]R flush_numbers: [5320, 5320, 5320, 5320, 5320]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.508877803Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"downstairs","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6932025-01-11T06:12:19.383Z{"msg":"[2]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.508897954Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"downstairs","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6942025-01-11T06:12:19.383Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.508926544Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"downstairs","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6952025-01-11T06:12:19.384Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.508951404Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"downstairs","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6962025-01-11T06:12:19.384Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.508964734Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"downstairs","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6972025-01-11T06:12:19.384Z{"msg":"Next flush: 5321","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.509010345Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"downstairs","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6982025-01-11T06:12:19.384Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.509026365Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"":"downstairs","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
6992025-01-11T06:12:19.384Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.509039185Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
7002025-01-11T06:12:19.384Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.509058955Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"0","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
7012025-01-11T06:12:19.384Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.509072425Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"1","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
7022025-01-11T06:12:19.384Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.509092686Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"client":"2","":"downstairs","session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
7032025-01-11T06:12:19.384Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.509106056Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
7042025-01-11T06:12:19.384Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.509125246Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
7052025-01-11T06:12:19.384Z{"msg":"258d6810-2a67-4a2a-9048-1d11597b14df is now active with session: 8a46df86-4105-47a5-bec1-811e292510a7","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.509138076Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
7062025-01-11T06:12:19.384Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.509156786Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244,"session_id":"8a46df86-4105-47a5-bec1-811e292510a7"}
7072025-01-11T06:12:19.384Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:17.509168897Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244}
7082025-01-11T06:12:19.384ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
7092025-01-11T06:12:19.387ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
7102025-01-11T06:12:19.387ZRead and Verify all blocks (0..100 range:true)
7112025-01-11T06:12:19.460ZWrote out file "/var/tmp/test_up-build/verify_file"
7122025-01-11T06:12:19.460ZVerify test completed
7132025-01-11T06:12:19.463ZWrote out file "/var/tmp/test_up-build/verify_file"
7142025-01-11T06:12:19.463ZCLIENT: Tests done. All submitted work has been ACK'd
7152025-01-11T06:12:19.463Z----------------------------------------------------------------
7162025-01-11T06:12:19.463Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
7172025-01-11T06:12:19.463ZStates: Active Active Active
7182025-01-11T06:12:19.463ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
7192025-01-11T06:12:19.463Z 1000 Acked Read 50 Done Done Done false
7202025-01-11T06:12:19.463Z STATES DS:0 DS:1 DS:2 TOTAL
7212025-01-11T06:12:19.463Z Sent 0 0 0 0
7222025-01-11T06:12:19.463Z Done 1 1 1 3
7232025-01-11T06:12:19.463Z Skipped 0 0 0 0
7242025-01-11T06:12:19.463Z Error 0 0 0 0
7252025-01-11T06:12:19.463ZLast Flush: 0 0 0
7262025-01-11T06:12:19.463ZDownstairs last five completed:
7272025-01-11T06:12:19.463ZUpstairs last five completed: 1000
7282025-01-11T06:12:19.463Z----------------------------------------------------------------
7292025-01-11T06:12:19.463Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
7302025-01-11T06:12:19.463ZStates: Active Active Active
7312025-01-11T06:12:19.463ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
7322025-01-11T06:12:19.463Z 1000 Acked Read 50 Done Done Done false
7332025-01-11T06:12:19.463Z STATES DS:0 DS:1 DS:2 TOTAL
7342025-01-11T06:12:19.463Z Sent 0 0 0 0
7352025-01-11T06:12:19.463Z Done 1 1 1 3
7362025-01-11T06:12:19.463Z Skipped 0 0 0 0
7372025-01-11T06:12:19.463Z Error 0 0 0 0
7382025-01-11T06:12:19.464ZLast Flush: 0 0 0
7392025-01-11T06:12:19.464ZDownstairs last five completed:
7402025-01-11T06:12:19.464ZUpstairs last five completed: 1000
7412025-01-11T06:12:19.464ZCLIENT: Up:0 ds:2 act:6
7422025-01-11T06:12:23.462Z----------------------------------------------------------------
7432025-01-11T06:12:23.462Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
7442025-01-11T06:12:23.462ZDownstairs last five completed: 1001 1000
7452025-01-11T06:12:23.462ZUpstairs last five completed: 1001 1000
7462025-01-11T06:12:23.462Z----------------------------------------------------------------
7472025-01-11T06:12:23.462Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
7482025-01-11T06:12:23.462ZDownstairs last five completed: 1001 1000
7492025-01-11T06:12:23.462ZUpstairs last five completed: 1001 1000
7502025-01-11T06:12:23.465ZCLIENT: Up:0 ds:0 act:6
7512025-01-11T06:12:23.465ZCLIENT: All crucible jobs finished, exiting program
7522025-01-11T06:12:23.465Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:12:21.58993986Z","hostname":"w-01JH9ZCR5JGH8B3VERD66A2P79","pid":1244}
7532025-01-11T06:12:23.473ZRepair part 2 passed
7542025-01-11T06:12:23.475Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
7552025-01-11T06:12:23.491ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
7562025-01-11T06:12:23.491Z 0 000-009 32 32 32 3998 3998 3998 F F F
7572025-01-11T06:12:23.491Z 1 010-019 32 32 32 3998 3998 3998 F F F
7582025-01-11T06:12:23.491Z 2 020-029 32 32 32 3998 3998 3998 F F F
7592025-01-11T06:12:23.491Z 3 030-039 32 32 32 3998 3998 3998 F F F
7602025-01-11T06:12:23.491Z 4 040-049 32 32 32 3998 3998 3998 F F F
7612025-01-11T06:12:23.494ZMax gen: 32, Max flush: 3998
7622025-01-11T06:12:23.496Zdump test passed
7632025-01-11T06:12:23.499Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
7642025-01-11T06:12:23.513Z Extent 1
7652025-01-11T06:12:23.516ZGEN 32 32 32
7662025-01-11T06:12:23.516ZFLUSH_ID 3998 3998 3998
7672025-01-11T06:12:23.516ZDIRTY
7682025-01-11T06:12:23.516Z
7692025-01-11T06:12:23.516ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
7702025-01-11T06:12:23.516Z010 A A A A A A
7712025-01-11T06:12:23.518Z011 A A A A A A
7722025-01-11T06:12:23.519Z012 A A A A A A
7732025-01-11T06:12:23.523Z013 A A A A A A
7742025-01-11T06:12:23.523Z014 A A A A A A
7752025-01-11T06:12:23.523Z015 A A A A A A
7762025-01-11T06:12:23.525Z016 A A A A A A
7772025-01-11T06:12:23.528Z017 A A A A A A
7782025-01-11T06:12:23.528Z018 A A A A A A
7792025-01-11T06:12:23.530Z019 A A A A A A
7802025-01-11T06:12:23.533Zdump extent test passed
7812025-01-11T06:12:23.536Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
7822025-01-11T06:12:23.551ZExtent 2 Block in extent 0 Actual block 20
7832025-01-11T06:12:23.551Z
7842025-01-11T06:12:23.554Z DATA SHA256 VER
7852025-01-11T06:12:23.554Z------ ---------------------------------------------------------------- ---
7862025-01-11T06:12:23.556Z 0 68b97a22c6937dacda06955a743385e8030bda5715041432ea0c4619e8079eb5 A
7872025-01-11T06:12:23.556Z 1 68b97a22c6937dacda06955a743385e8030bda5715041432ea0c4619e8079eb5 A
7882025-01-11T06:12:23.556Z 2 68b97a22c6937dacda06955a743385e8030bda5715041432ea0c4619e8079eb5 A
7892025-01-11T06:12:23.557Z
7902025-01-11T06:12:23.557ZNONCES 0 1 2 DIFF
7912025-01-11T06:12:23.557Z------ ------------------------ ------------------------ ------------------------ -----
7922025-01-11T06:12:23.557Z 0 3cf70ba13287695040fba2fa 3cf70ba13287695040fba2fa 3cf70ba13287695040fba2fa
7932025-01-11T06:12:23.557Z
7942025-01-11T06:12:23.557Z TAGS 0 1 2 DIFF
7952025-01-11T06:12:23.557Z------ -------------------------------- -------------------------------- -------------------------------- -----
7962025-01-11T06:12:23.557Z 0 c86b34651087472fd2b6057c17382e9a c86b34651087472fd2b6057c17382e9a c86b34651087472fd2b6057c17382e9a
7972025-01-11T06:12:23.557Z
7982025-01-11T06:12:23.557ZHASHES 0 1 2 DIFF
7992025-01-11T06:12:23.557Z------ ---------------- ---------------- ---------------- -----
8002025-01-11T06:12:23.557Z
8012025-01-11T06:12:23.560Zdump block test passed
8022025-01-11T06:12:23.560ZInitial upstairs tests have completed, stopping all downstairs
8032025-01-11T06:12:28.626ZCreating 7 larger downstairs regions
8042025-01-11T06:12:28.819ZStarting 7 downstairs
8052025-01-11T06:12:33.826Zdsc restarted at PID: 1260
8062025-01-11T06:12:34.247ZNow do the replace-reconcile test
8072025-01-11T06:12:34.308ZUsing 8870 for the replacement port
8082025-01-11T06:15:11.739ZCompleted test: replace-reconcile
8092025-01-11T06:15:11.742ZNow do the replace-before-active test
8102025-01-11T06:18:45.445ZCompleted test: replace-before-active
8112025-01-11T06:18:45.445ZAll tests have completed, stopping all downstairs
8122025-01-11T06:18:45.506Z
8132025-01-11T06:18:45.506ZAll Tests have passed
8142025-01-11T06:18:45.509Z13:23 Test duration
8152025-01-11T06:18:45.509Z
8162025-01-11T06:18:45.509Zreal 13:22.879509384
8172025-01-11T06:18:45.509Zuser 18:27.504922924
8182025-01-11T06:18:45.509Zsys 26.448280240
8192025-01-11T06:18:45.509Ztrap 0.547320905
8202025-01-11T06:18:45.509Ztflt 0.023782801
8212025-01-11T06:18:45.509Zdflt 0.102027980
8222025-01-11T06:18:45.510Zkflt 0.001623336
8232025-01-11T06:18:45.510Zlock 25:33:50.121784292
8242025-01-11T06:18:45.510Zslp 1:14:20.349329780
8252025-01-11T06:18:45.510Zlat 2:03.785754124
8262025-01-11T06:18:45.510Zstop 0.210515895
8272025-01-11T06:18:45.510Z+ echo 'test-up-2region-encrypted ends'
8282025-01-11T06:18:45.510Ztest-up-2region-encrypted ends
8292025-01-11T06:18:50.512Zprocess exited: duration 811128 ms, exit code 0
8302025-01-11T06:18:50.512Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
8312025-01-11T06:19:50.552Zexec warning: : stdout descriptor may be held open by a background process; giving up!
8322025-01-11T06:19:50.552Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
8332025-01-11T06:19:50.563Zfound 17 output files
8342025-01-11T06:19:50.564Zuploading: /tmp/test_up-build/dsc-out.txt (48907 bytes)
8352025-01-11T06:19:51.580Zuploaded: /tmp/test_up-build/dsc-out.txt
8362025-01-11T06:19:51.582Zuploading: /tmp/test_up-build/test_up_out.txt (11934535 bytes)
8372025-01-11T06:19:52.796Zuploaded: /tmp/test_up-build/test_up_out.txt
8382025-01-11T06:19:52.799Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (147234 bytes)
8392025-01-11T06:19:53.817Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
8402025-01-11T06:19:53.819Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (144652 bytes)
8412025-01-11T06:19:54.837Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
8422025-01-11T06:19:54.840Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (251051 bytes)
8432025-01-11T06:19:55.861Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
8442025-01-11T06:19:55.864Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (139424 bytes)
8452025-01-11T06:19:56.880Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
8462025-01-11T06:19:56.882Zuploading: /tmp/test_up-build/dsc/downstairs-8850.txt (2501 bytes)
8472025-01-11T06:19:57.890Zuploaded: /tmp/test_up-build/dsc/downstairs-8850.txt
8482025-01-11T06:19:57.892Zuploading: /tmp/test_up-build/dsc/downstairs-8860.txt (33788 bytes)
8492025-01-11T06:19:58.902Zuploaded: /tmp/test_up-build/dsc/downstairs-8860.txt
8502025-01-11T06:19:58.904Zuploading: /tmp/test_up-build/dsc/downstairs-8870.txt (735762 bytes)
8512025-01-11T06:19:59.940Zuploaded: /tmp/test_up-build/dsc/downstairs-8870.txt
8522025-01-11T06:19:59.942Zuploading: /tmp/debug/df.txt (1270 bytes)
8532025-01-11T06:20:00.949Zuploaded: /tmp/debug/df.txt
8542025-01-11T06:20:00.952Zuploading: /tmp/debug/dtrace.txt (991769 bytes)
8552025-01-11T06:20:01.994Zuploaded: /tmp/debug/dtrace.txt
8562025-01-11T06:20:01.997Zuploading: /tmp/debug/iostat.txt (260460 bytes)
8572025-01-11T06:20:02.014Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 260460 -> 264060
8582025-01-11T06:20:03.021Zuploaded: /tmp/debug/iostat.txt
8592025-01-11T06:20:03.024Zuploading: /tmp/debug/mpstat.txt (632786 bytes)
8602025-01-11T06:20:03.053Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 632786 -> 642263
8612025-01-11T06:20:04.062Zuploaded: /tmp/debug/mpstat.txt
8622025-01-11T06:20:04.062Zuploading: /tmp/debug/paging.txt (113566 bytes)
8632025-01-11T06:20:04.075Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 113566 -> 115492
8642025-01-11T06:20:05.081Zuploaded: /tmp/debug/paging.txt
8652025-01-11T06:20:05.084Zuploading: /tmp/debug/prstat.txt (1228888 bytes)
8662025-01-11T06:20:05.126Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 1228888 -> 1246963
8672025-01-11T06:20:06.132Zuploaded: /tmp/debug/prstat.txt
8682025-01-11T06:20:06.135Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
8692025-01-11T06:20:07.141Zuploaded: /tmp/debug/psrinfo.txt
8702025-01-11T06:20:07.143Zuploading: /tmp/debug/upstairs-info.txt (248376 bytes)
8712025-01-11T06:20:08.162Zuploaded: /tmp/debug/upstairs-info.txt