01JN5HDP8YTVJDBWB0Q214J4PN: test-up-2region-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01JN5HE4Z4JC3F7868119KX7KD

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12025-02-28T05:59:24.372Zjob dependencies complete; ready to run (waiting for 13 m 25 s)
22025-02-28T06:00:30.906Zjob assigned to worker 01JN5J6KH8PE1MH4X3YMYEWZDH [factory aws, i-07603921de5f5823e] (queued for 1 m 6 s)
32025-02-28T06:00:39.338Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42025-02-28T06:00:40.495Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52025-02-28T06:00:40.495Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62025-02-28T06:00:41.544Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72025-02-28T06:00:41.544Zdownloading input: /input/build/work/bins/crutest.gz
82025-02-28T06:00:42.723Zdownloaded input: /input/build/work/bins/crutest.gz
92025-02-28T06:00:42.724Zdownloading input: /input/build/work/bins/dsc.gz
102025-02-28T06:00:43.437Zdownloaded input: /input/build/work/bins/dsc.gz
112025-02-28T06:00:43.443Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122025-02-28T06:00:43.454Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132025-02-28T06:00:43.454Zdownloading input: /input/build/work/scripts/test_ds.sh
142025-02-28T06:00:43.476Zdownloaded input: /input/build/work/scripts/test_ds.sh
152025-02-28T06:00:43.476Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162025-02-28T06:00:43.498Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172025-02-28T06:00:43.498Zdownloading input: /input/build/work/scripts/test_repair.sh
182025-02-28T06:00:43.520Zdownloaded input: /input/build/work/scripts/test_repair.sh
192025-02-28T06:00:43.520Zdownloading input: /input/build/work/scripts/test_replay.sh
202025-02-28T06:00:43.538Zdownloaded input: /input/build/work/scripts/test_replay.sh
212025-02-28T06:00:43.539Zdownloading input: /input/build/work/scripts/test_up.sh
222025-02-28T06:00:43.574Zdownloaded input: /input/build/work/scripts/test_up.sh
232025-02-28T06:00:43.574Zdownloading input: /input/build/work/scripts/upstairs_info.d
242025-02-28T06:00:43.596Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252025-02-28T06:00:43.599Zdownloading input: /input/build/tmp/cargo-test-out.log
262025-02-28T06:00:44.284Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272025-02-28T06:00:44.284Zstarting task 0: "setup"
282025-02-28T06:00:44.291Z++ uname -s
292025-02-28T06:00:44.295Z+ kern=SunOS
302025-02-28T06:00:44.295Z+ build_user=build
312025-02-28T06:00:44.295Z+ build_uid=12345
322025-02-28T06:00:44.295Z+ work_dir=/work
332025-02-28T06:00:44.295Z+ input_dir=/input
342025-02-28T06:00:44.295Z+ [[ 0 == 12345 ]]
352025-02-28T06:00:44.295Z+ case "$kern" in
362025-02-28T06:00:44.295Z+ groupadd -g 12345 build
372025-02-28T06:00:44.300Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382025-02-28T06:00:46.306Z+ zfs create -o mountpoint=/work rpool/work
392025-02-28T06:00:46.460Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402025-02-28T06:00:46.464Z+ home_fs=zfs
412025-02-28T06:00:46.464Z+ [[ zfs == autofs ]]
422025-02-28T06:00:46.464Z+ mkdir -p /home/build
432025-02-28T06:00:46.464Z+ chown build:build /home/build /work
442025-02-28T06:00:48.468Z+ chmod 0700 /home/build /work
452025-02-28T06:00:48.472Zprocess exited: duration 4185 ms, exit code 0
 
462025-02-28T06:00:48.495Zstarting task 1: "authentication"
472025-02-28T06:00:48.514Zprocess exited: duration 18 ms, exit code 0
 
482025-02-28T06:00:48.522Zstarting task 2: "build"
492025-02-28T06:00:48.526Z+ banner cores
502025-02-28T06:00:48.529Z
512025-02-28T06:00:48.529Z #### #### ##### ###### ####
522025-02-28T06:00:48.529Z # # # # # # # #
532025-02-28T06:00:48.529Z # # # # # ##### ####
542025-02-28T06:00:48.529Z # # # ##### # #
552025-02-28T06:00:48.529Z # # # # # # # # #
562025-02-28T06:00:48.529Z #### #### # # ###### ####
572025-02-28T06:00:48.529Z
582025-02-28T06:00:48.533Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
592025-02-28T06:00:48.541Z+ banner unpack
602025-02-28T06:00:48.555Z
612025-02-28T06:00:48.555Z # # # # ##### ## #### # #
622025-02-28T06:00:48.555Z # # ## # # # # # # # # #
632025-02-28T06:00:48.555Z # # # # # # # # # # ####
642025-02-28T06:00:48.555Z # # # # # ##### ###### # # #
652025-02-28T06:00:48.556Z # # # ## # # # # # # #
662025-02-28T06:00:48.556Z #### # # # # # #### # #
672025-02-28T06:00:48.556Z
682025-02-28T06:00:48.556Z+ mkdir -p /var/tmp/bins
692025-02-28T06:00:48.556Z+ for t in "$input/bins/"*.gz
702025-02-28T06:00:48.556Z++ basename /input/build/work/bins/crucible-downstairs.gz
712025-02-28T06:00:48.556Z+ b=crucible-downstairs.gz
722025-02-28T06:00:48.556Z+ b=crucible-downstairs
732025-02-28T06:00:48.556Z+ gunzip
742025-02-28T06:00:50.809Z+ chmod +x /var/tmp/bins/crucible-downstairs
752025-02-28T06:00:50.825Z+ for t in "$input/bins/"*.gz
762025-02-28T06:00:50.825Z++ basename /input/build/work/bins/crucible-hammer.gz
772025-02-28T06:00:50.825Z+ b=crucible-hammer.gz
782025-02-28T06:00:50.825Z+ b=crucible-hammer
792025-02-28T06:00:50.825Z+ gunzip
802025-02-28T06:00:52.937Z+ chmod +x /var/tmp/bins/crucible-hammer
812025-02-28T06:00:52.941Z+ for t in "$input/bins/"*.gz
822025-02-28T06:00:52.941Z++ basename /input/build/work/bins/crutest.gz
832025-02-28T06:00:52.946Z+ b=crutest.gz
842025-02-28T06:00:52.946Z+ b=crutest
852025-02-28T06:00:52.946Z+ gunzip
862025-02-28T06:00:55.341Z+ chmod +x /var/tmp/bins/crutest
872025-02-28T06:00:55.352Z+ for t in "$input/bins/"*.gz
882025-02-28T06:00:55.352Z++ basename /input/build/work/bins/dsc.gz
892025-02-28T06:00:55.352Z+ b=dsc.gz
902025-02-28T06:00:55.352Z+ b=dsc
912025-02-28T06:00:55.352Z+ gunzip
922025-02-28T06:00:56.709Z+ chmod +x /var/tmp/bins/dsc
932025-02-28T06:00:56.720Z+ export BINDIR=/var/tmp/bins
942025-02-28T06:00:56.720Z+ BINDIR=/var/tmp/bins
952025-02-28T06:00:56.720Z+ jobpid=1148
962025-02-28T06:00:56.720Z+ echo 'Setup debug logging'
972025-02-28T06:00:56.720ZSetup debug logging
982025-02-28T06:00:56.720Z+ mkdir /tmp/debug
992025-02-28T06:00:56.720Z+ sleep 3600
1002025-02-28T06:00:56.720Z+ psrinfo -v
1012025-02-28T06:00:56.720Z+ df -h
1022025-02-28T06:00:56.731Z+ prstat -d d -mLc 1
1032025-02-28T06:00:56.731Z+ iostat -T d -xn 1
1042025-02-28T06:00:56.731Z+ mpstat -T d 1
1052025-02-28T06:00:56.738Z+ vmstat -T d -p 1
1062025-02-28T06:00:56.738Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1072025-02-28T06:00:56.738Z+ banner test_up_2r_encrypted
1082025-02-28T06:00:56.738Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1092025-02-28T06:00:56.748Z #####
1102025-02-28T06:00:56.748Z ##### ###### #### ##### # # ##### # # #####
1112025-02-28T06:00:56.748Z # # # # # # # # # # #
1122025-02-28T06:00:56.748Z # ##### #### # # # # # ##### # #
1132025-02-28T06:00:56.748Z # # # # # # ##### # #####
1142025-02-28T06:00:56.748Z # # # # # # # # # # #
1152025-02-28T06:00:56.748Z # ###### #### # ####### #### # ####### ####### # #
1162025-02-28T06:00:56.748Z
1172025-02-28T06:00:56.748Z+ ptime -m bash /input/build/work/scripts/test_up.sh -r 2 -N encrypted
1182025-02-28T06:00:56.749Z/input/build/work
1192025-02-28T06:00:56.757ZUsing 2 region sets
1202025-02-28T06:00:56.757ZTurn off color for downstairs dump
1212025-02-28T06:00:56.774ZUpstairs using key: 0zcV00IJ9kc5IAC8Oj/kAilIhUXe3EG2eWdkyXyg+TI=
1222025-02-28T06:00:56.785Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1232025-02-28T06:00:56.785ZCreating 6 downstairs regions
1242025-02-28T06:00:57.188ZStarting 6 downstairs
1252025-02-28T06:00:57.206Zdsc started at PID: 1196
1262025-02-28T06:01:03.483ZDisable automatic restart on all downstairs
1272025-02-28T06:01:03.483Z
1282025-02-28T06:01:03.483ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1292025-02-28T06:01:03.483ZRunning test: span
1302025-02-28T06:01:06.746ZCompleted test: span
1312025-02-28T06:01:11.754ZRunning test: big
1322025-02-28T06:01:16.334ZCompleted test: big
1332025-02-28T06:01:21.341ZRunning test: dep
1342025-02-28T06:01:54.970ZCompleted test: dep
1352025-02-28T06:01:59.989ZRunning test: balloon
1362025-02-28T06:06:12.498ZCompleted test: balloon
1372025-02-28T06:06:17.505ZRunning test: deactivate
1382025-02-28T06:07:12.179ZCompleted test: deactivate
1392025-02-28T06:07:17.186ZRunning hammer
1402025-02-28T06:07:29.712ZRun repair tests
1412025-02-28T06:07:29.715Z/var/tmp/bins/crutest fill -g 31 -q --key 0zcV00IJ9kc5IAC8Oj/kAilIhUXe3EG2eWdkyXyg+TI= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1422025-02-28T06:07:29.773Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.890044989Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"task":"crutest"}
1432025-02-28T06:07:29.776Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.890558059Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"task":"crutest"}
1442025-02-28T06:07:29.776Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.892255884Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"task":"crutest"}
1452025-02-28T06:07:29.777Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.892442908Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239}
1462025-02-28T06:07:29.777Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0c103f9ae2997264530dbd443fadbdf83878dac4\",\n git_commit_timestamp: \"2025-02-28T05:45:10.000000000Z\",\n git_branch: \"renovate/uuid-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.892493719Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239}
1472025-02-28T06:07:29.777Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.8925244Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239}
1482025-02-28T06:07:29.777Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.89254548Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239}
1492025-02-28T06:07:29.777Z{"msg":"Crucible 966b669c-e780-4929-a4e1-bf04f8854fcc has session id: c47d525e-a593-4bdf-ac08-9d17d361026d","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.893103452Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1502025-02-28T06:07:29.777Z{"msg":"Upstairs opts: Upstairs UUID: 966b669c-e780-4929-a4e1-bf04f8854fcc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.893144923Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1512025-02-28T06:07:29.777Z{"msg":"Crucible stats registered with UUID: 966b669c-e780-4929-a4e1-bf04f8854fcc","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.893159443Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1522025-02-28T06:07:29.777Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-28T06:07:28.893177793Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1532025-02-28T06:07:29.781Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.895243466Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"task":"crutest"}
1542025-02-28T06:07:29.781Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.895331088Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239}
1552025-02-28T06:07:29.781Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0c103f9ae2997264530dbd443fadbdf83878dac4\",\n git_commit_timestamp: \"2025-02-28T05:45:10.000000000Z\",\n git_branch: \"renovate/uuid-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.895368588Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239}
1562025-02-28T06:07:29.781Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.895397069Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239}
1572025-02-28T06:07:29.781Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.895419569Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239}
1582025-02-28T06:07:29.781Z{"msg":"Crucible 966b669c-e780-4929-a4e1-bf04f8854fcc has session id: 9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.895756816Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
1592025-02-28T06:07:29.781Z{"msg":"Upstairs opts: Upstairs UUID: 966b669c-e780-4929-a4e1-bf04f8854fcc, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.895807457Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
1602025-02-28T06:07:29.781Z{"msg":"Crucible stats registered with UUID: 966b669c-e780-4929-a4e1-bf04f8854fcc","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.895831798Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
1612025-02-28T06:07:29.781Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-28T06:07:28.895854708Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
1622025-02-28T06:07:29.781Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.896457191Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"task":"crutest"}
1632025-02-28T06:07:29.781Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.896489551Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239}
1642025-02-28T06:07:29.781Z{"msg":"966b669c-e780-4929-a4e1-bf04f8854fcc active request set","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.896577373Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1652025-02-28T06:07:29.783Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.896604344Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"client":"0","":"downstairs","session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1662025-02-28T06:07:29.783Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.896628354Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"client":"0","":"downstairs","session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1672025-02-28T06:07:29.783Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.896662095Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"client":"1","":"downstairs","session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1682025-02-28T06:07:29.783Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.896686065Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"client":"1","":"downstairs","session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1692025-02-28T06:07:29.783Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.896707586Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"client":"2","":"downstairs","session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1702025-02-28T06:07:29.783Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.896731386Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"client":"2","":"downstairs","session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1712025-02-28T06:07:29.783Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.896762467Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"io task","client":"2","":"downstairs","session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1722025-02-28T06:07:29.783Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.896788047Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"io task","client":"0","":"downstairs","session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1732025-02-28T06:07:29.783Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.896820538Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"io task","client":"1","":"downstairs","session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1742025-02-28T06:07:29.783Z{"msg":"ds_connection connected from Ok(127.0.0.1:56409)","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.896854009Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"io task","client":"0","":"downstairs","session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1752025-02-28T06:07:29.783Z{"msg":"ds_connection connected from Ok(127.0.0.1:35348)","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.896885889Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"io task","client":"2","":"downstairs","session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1762025-02-28T06:07:29.784Z{"msg":"ds_connection connected from Ok(127.0.0.1:54011)","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.89692397Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"io task","client":"1","":"downstairs","session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1772025-02-28T06:07:29.784Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.897542563Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"client":"2","":"downstairs","session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1782025-02-28T06:07:29.784Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.897680096Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"client":"1","":"downstairs","session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1792025-02-28T06:07:29.784Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.897721157Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"client":"0","":"downstairs","session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1802025-02-28T06:07:29.792Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.898208437Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"client":"2","":"downstairs","session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1812025-02-28T06:07:29.793Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.898245857Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"client":"1","":"downstairs","session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1822025-02-28T06:07:29.793Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.898410551Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"client":"0","":"downstairs","session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1832025-02-28T06:07:29.793Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.898666936Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1842025-02-28T06:07:29.793Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.898813789Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1852025-02-28T06:07:29.793Z{"msg":"[0]R flush_numbers: [3999, 3999, 3999, 3999, 3999]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.89884486Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"downstairs","":"downstairs","session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1862025-02-28T06:07:29.793Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.89885988Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"downstairs","":"downstairs","session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1872025-02-28T06:07:29.793Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.898880831Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"downstairs","":"downstairs","session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1882025-02-28T06:07:29.793Z{"msg":"[1]R flush_numbers: [3999, 3999, 3999, 3999, 3999]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.898906411Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"downstairs","":"downstairs","session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1892025-02-28T06:07:29.793Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.898921102Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"downstairs","":"downstairs","session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1902025-02-28T06:07:29.793Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.898946792Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"downstairs","":"downstairs","session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1912025-02-28T06:07:29.793Z{"msg":"[2]R flush_numbers: [3999, 3999, 3999, 3999, 3999]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.898960202Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"downstairs","":"downstairs","session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1922025-02-28T06:07:29.793Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.898990603Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"downstairs","":"downstairs","session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1932025-02-28T06:07:29.793Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.899013473Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"downstairs","":"downstairs","session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1942025-02-28T06:07:29.793Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.899026824Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"downstairs","":"downstairs","session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1952025-02-28T06:07:29.793Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.899041234Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"downstairs","":"downstairs","session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1962025-02-28T06:07:29.793Z{"msg":"Next flush: 4000","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.899053974Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"downstairs","":"downstairs","session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1972025-02-28T06:07:29.793Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.899078355Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"downstairs","":"downstairs","session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1982025-02-28T06:07:29.793Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.899091365Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
1992025-02-28T06:07:29.793Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.899110245Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"client":"0","":"downstairs","session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
2002025-02-28T06:07:29.793Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.899123646Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"client":"1","":"downstairs","session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
2012025-02-28T06:07:29.793Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.899143396Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"client":"2","":"downstairs","session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
2022025-02-28T06:07:29.793Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.899156306Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
2032025-02-28T06:07:29.793Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.899167956Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
2042025-02-28T06:07:29.793Z{"msg":"966b669c-e780-4929-a4e1-bf04f8854fcc is now active with session: c47d525e-a593-4bdf-ac08-9d17d361026d","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.899187377Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
2052025-02-28T06:07:29.793Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.899199107Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"session_id":"c47d525e-a593-4bdf-ac08-9d17d361026d"}
2062025-02-28T06:07:29.793Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.899222818Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239}
2072025-02-28T06:07:29.793Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.899234068Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239}
2082025-02-28T06:07:29.793Z{"msg":"966b669c-e780-4929-a4e1-bf04f8854fcc active request set","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.899255788Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2092025-02-28T06:07:29.794Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.899279979Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"client":"0","":"downstairs","session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2102025-02-28T06:07:29.794Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.899305409Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"client":"0","":"downstairs","session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2112025-02-28T06:07:29.794Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.89933548Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"client":"1","":"downstairs","session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2122025-02-28T06:07:29.794Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.89936004Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"client":"1","":"downstairs","session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2132025-02-28T06:07:29.794Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.899382441Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"client":"2","":"downstairs","session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2142025-02-28T06:07:29.794Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.899422972Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"client":"2","":"downstairs","session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2152025-02-28T06:07:29.794Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.899537024Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"io task","client":"2","":"downstairs","session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2162025-02-28T06:07:29.794Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.899586245Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"io task","client":"0","":"downstairs","session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2172025-02-28T06:07:29.794Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.899630516Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"io task","client":"1","":"downstairs","session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2182025-02-28T06:07:29.794Z{"msg":"ds_connection connected from Ok(127.0.0.1:52444)","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.899663627Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"io task","client":"2","":"downstairs","session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2192025-02-28T06:07:29.794Z{"msg":"ds_connection connected from Ok(127.0.0.1:64517)","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.899706458Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"io task","client":"0","":"downstairs","session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2202025-02-28T06:07:29.794Z{"msg":"ds_connection connected from Ok(127.0.0.1:35154)","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.899723448Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"io task","client":"1","":"downstairs","session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2212025-02-28T06:07:29.794Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.899744978Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"client":"0","":"downstairs","session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2222025-02-28T06:07:29.794Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.899891121Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"client":"1","":"downstairs","session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2232025-02-28T06:07:29.794Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.899933102Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"client":"2","":"downstairs","session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2242025-02-28T06:07:29.794Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.900332591Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"client":"0","":"downstairs","session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2252025-02-28T06:07:29.794Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.900444193Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"client":"1","":"downstairs","session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2262025-02-28T06:07:29.794Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.900609896Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"client":"2","":"downstairs","session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2272025-02-28T06:07:29.795Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.900742089Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2282025-02-28T06:07:29.795Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.900858631Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2292025-02-28T06:07:29.795Z{"msg":"[0]R flush_numbers: [5321, 5320, 5320, 5318, 5320]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.900959703Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"downstairs","":"downstairs","session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2302025-02-28T06:07:29.795Z{"msg":"[0]R generation: [14, 14, 14, 13, 14]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.900986574Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"downstairs","":"downstairs","session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2312025-02-28T06:07:29.795Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.901027555Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"downstairs","":"downstairs","session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2322025-02-28T06:07:29.795Z{"msg":"[1]R flush_numbers: [5321, 5320, 5320, 5318, 5320]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.901042095Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"downstairs","":"downstairs","session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2332025-02-28T06:07:29.795Z{"msg":"[1]R generation: [14, 14, 14, 13, 14]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.901055495Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"downstairs","":"downstairs","session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2342025-02-28T06:07:29.795Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.901075276Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"downstairs","":"downstairs","session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2352025-02-28T06:07:29.795Z{"msg":"[2]R flush_numbers: [5321, 5320, 5320, 5318, 5320]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.901088656Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"downstairs","":"downstairs","session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2362025-02-28T06:07:29.795Z{"msg":"[2]R generation: [14, 14, 14, 13, 14]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.901101486Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"downstairs","":"downstairs","session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2372025-02-28T06:07:29.795Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
2382025-02-28T06:07:29.795Z,"time":"2025-02-28T06:07:28.901115267Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"downstairs","":"downstairs","session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2392025-02-28T06:07:29.795Z{"msg":"Max found gen is 15","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.901144707Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"downstairs","":"downstairs","session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2402025-02-28T06:07:29.795Z{"msg":"Generation requested: 31 >= found:15","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.901157597Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"downstairs","":"downstairs","session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2412025-02-28T06:07:29.795Z{"msg":"Next flush: 5322","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.901177168Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"downstairs","":"downstairs","session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2422025-02-28T06:07:29.795Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.901190008Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"":"downstairs","":"downstairs","session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2432025-02-28T06:07:29.795Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.901202838Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2442025-02-28T06:07:29.795Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.901214499Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"client":"0","":"downstairs","session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2452025-02-28T06:07:29.795Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.901226989Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"client":"1","":"downstairs","session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2462025-02-28T06:07:29.795Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.901255819Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"client":"2","":"downstairs","session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2472025-02-28T06:07:29.795Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.90127116Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2482025-02-28T06:07:29.795Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.90129042Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2492025-02-28T06:07:29.795Z{"msg":"966b669c-e780-4929-a4e1-bf04f8854fcc is now active with session: 9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.90130346Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2502025-02-28T06:07:29.795Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.901315011Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239,"session_id":"9aaabe30-2e27-4b2a-a54c-9c9745c5bdf1"}
2512025-02-28T06:07:29.795Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:28.901333731Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239}
2522025-02-28T06:07:29.795ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
2532025-02-28T06:07:29.795ZFill test
2542025-02-28T06:07:29.839ZRead and Verify all blocks (0..100 range:false)
2552025-02-28T06:07:29.927ZWrote out file "/var/tmp/test_up-build/verify_file"
2562025-02-28T06:07:29.927ZCLIENT: Tests done. All submitted work has been ACK'd
2572025-02-28T06:07:29.930Z----------------------------------------------------------------
2582025-02-28T06:07:29.931Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2592025-02-28T06:07:29.931ZStates: Active Active Active
2602025-02-28T06:07:29.931ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2612025-02-28T06:07:29.931Z 1002 Acked Read 50 Done Done Done false
2622025-02-28T06:07:29.931Z STATES DS:0 DS:1 DS:2 TOTAL
2632025-02-28T06:07:29.931Z Sent 0 0 0 0
2642025-02-28T06:07:29.931Z Done 1 1 1 3
2652025-02-28T06:07:29.931Z Skipped 0 0 0 0
2662025-02-28T06:07:29.931Z Error 0 0 0 0
2672025-02-28T06:07:29.931ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2682025-02-28T06:07:29.932ZDownstairs last five completed: 1001 1000
2692025-02-28T06:07:29.932ZUpstairs last five completed: 1002 1001 1000
2702025-02-28T06:07:29.932Z----------------------------------------------------------------
2712025-02-28T06:07:29.932Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2722025-02-28T06:07:29.932ZStates: Active Active Active
2732025-02-28T06:07:29.932ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2742025-02-28T06:07:29.932Z 1002 Acked Read 50 Done Done Done false
2752025-02-28T06:07:29.932Z STATES DS:0 DS:1 DS:2 TOTAL
2762025-02-28T06:07:29.932Z Sent 0 0 0 0
2772025-02-28T06:07:29.932Z Done 1 1 1 3
2782025-02-28T06:07:29.932Z Skipped 0 0 0 0
2792025-02-28T06:07:29.932Z Error 0 0 0 0
2802025-02-28T06:07:29.932ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2812025-02-28T06:07:29.932ZDownstairs last five completed: 1001 1000
2822025-02-28T06:07:29.932ZUpstairs last five completed: 1002 1001 1000
2832025-02-28T06:07:29.932ZCLIENT: Up:0 ds:2 act:6
2842025-02-28T06:07:33.928Z----------------------------------------------------------------
2852025-02-28T06:07:33.929Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2862025-02-28T06:07:33.929ZDownstairs last five completed: 1003 1002 1001 1000
2872025-02-28T06:07:33.929ZUpstairs last five completed: 1003 1002 1001 1000
2882025-02-28T06:07:33.929Z----------------------------------------------------------------
2892025-02-28T06:07:33.929Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2902025-02-28T06:07:33.929ZDownstairs last five completed: 1003 1002 1001 1000
2912025-02-28T06:07:33.929ZUpstairs last five completed: 1003 1002 1001 1000
2922025-02-28T06:07:33.932ZCLIENT: Up:0 ds:0 act:6
2932025-02-28T06:07:33.932ZCLIENT: All crucible jobs finished, exiting program
2942025-02-28T06:07:33.932Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-28T06:07:33.045773043Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1239}
2952025-02-28T06:07:33.941ZRepair setup passed
2962025-02-28T06:07:33.944ZCopy the region for /var/tmp/test_up-build/8830
2972025-02-28T06:07:33.944Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2982025-02-28T06:07:33.947Z/var/tmp/bins/crutest fill -g 32 -q --key 0zcV00IJ9kc5IAC8Oj/kAilIhUXe3EG2eWdkyXyg+TI= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2992025-02-28T06:07:34.005Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.121606824Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"task":"crutest"}
3002025-02-28T06:07:34.008Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.122260827Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"task":"crutest"}
3012025-02-28T06:07:34.008Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.124009553Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"task":"crutest"}
3022025-02-28T06:07:34.010Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.124194647Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245}
3032025-02-28T06:07:34.010Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0c103f9ae2997264530dbd443fadbdf83878dac4\",\n git_commit_timestamp: \"2025-02-28T05:45:10.000000000Z\",\n git_branch: \"renovate/uuid-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.124243058Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245}
3042025-02-28T06:07:34.010Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.124265849Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245}
3052025-02-28T06:07:34.010Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.124295209Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245}
3062025-02-28T06:07:34.010Z{"msg":"Crucible 345efba9-2a9c-428b-a9ca-28d076226af9 has session id: 376a641b-6047-4d0f-af59-33fe2f088a86","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.124844561Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3072025-02-28T06:07:34.010Z{"msg":"Upstairs opts: Upstairs UUID: 345efba9-2a9c-428b-a9ca-28d076226af9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.124902632Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3082025-02-28T06:07:34.010Z{"msg":"Crucible stats registered with UUID: 345efba9-2a9c-428b-a9ca-28d076226af9","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.124929112Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3092025-02-28T06:07:34.010Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-28T06:07:33.124961453Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3102025-02-28T06:07:34.013Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.126979374Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"task":"crutest"}
3112025-02-28T06:07:34.013Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.127040206Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245}
3122025-02-28T06:07:34.013Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0c103f9ae2997264530dbd443fadbdf83878dac4\",\n git_commit_timestamp: \"2025-02-28T05:45:10.000000000Z\",\n git_branch: \"renovate/uuid-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.127070616Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245}
3132025-02-28T06:07:34.013Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.127096507Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245}
3142025-02-28T06:07:34.013Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.127114407Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245}
3152025-02-28T06:07:34.013Z{"msg":"Crucible 345efba9-2a9c-428b-a9ca-28d076226af9 has session id: 5509d32f-246e-4eb1-b220-a1061237a9ee","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.127452404Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3162025-02-28T06:07:34.013Z{"msg":"Upstairs opts: Upstairs UUID: 345efba9-2a9c-428b-a9ca-28d076226af9, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.127511436Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3172025-02-28T06:07:34.013Z{"msg":"Crucible stats registered with UUID: 345efba9-2a9c-428b-a9ca-28d076226af9","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.127598507Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3182025-02-28T06:07:34.013Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-28T06:07:33.127634668Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3192025-02-28T06:07:34.013Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.128185149Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"task":"crutest"}
3202025-02-28T06:07:34.014Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.1282212Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245}
3212025-02-28T06:07:34.014Z{"msg":"345efba9-2a9c-428b-a9ca-28d076226af9 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.128251671Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3222025-02-28T06:07:34.014Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.128285091Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"client":"0","":"downstairs","session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3232025-02-28T06:07:34.014Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.128318452Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"client":"0","":"downstairs","session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3242025-02-28T06:07:34.014Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.128353153Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"client":"1","":"downstairs","session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3252025-02-28T06:07:34.014Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.128382413Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"client":"1","":"downstairs","session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3262025-02-28T06:07:34.014Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.128427644Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"client":"2","":"downstairs","session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3272025-02-28T06:07:34.014Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.128457185Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"client":"2","":"downstairs","session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3282025-02-28T06:07:34.014Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.128493036Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"io task","client":"2","":"downstairs","session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3292025-02-28T06:07:34.014Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.128529686Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"io task","client":"0","":"downstairs","session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3302025-02-28T06:07:34.014Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.128551447Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"io task","client":"1","":"downstairs","session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3312025-02-28T06:07:34.014Z{"msg":"ds_connection connected from Ok(127.0.0.1:47577)","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.128582698Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"io task","client":"2","":"downstairs","session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3322025-02-28T06:07:34.015Z{"msg":"ds_connection connected from Ok(127.0.0.1:59060)","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.128605618Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"io task","client":"1","":"downstairs","session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3332025-02-28T06:07:34.015Z{"msg":"ds_connection connected from Ok(127.0.0.1:47833)","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.128756161Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"io task","client":"0","":"downstairs","session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3342025-02-28T06:07:34.015Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.129281212Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"client":"2","":"downstairs","session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3352025-02-28T06:07:34.015Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.129318973Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"client":"1","":"downstairs","session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3362025-02-28T06:07:34.015Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.129335403Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"client":"0","":"downstairs","session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3372025-02-28T06:07:34.015Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.129939346Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"client":"0","":"downstairs","session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3382025-02-28T06:07:34.019Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.130055628Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"client":"1","":"downstairs","session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3392025-02-28T06:07:34.019Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.130081668Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"client":"2","":"downstairs","session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3402025-02-28T06:07:34.019Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.130394285Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3412025-02-28T06:07:34.019Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.130488937Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3422025-02-28T06:07:34.019Z{"msg":"[0]R flush_numbers: [4000, 4000, 4000, 4000, 4000]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.13063065Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"downstairs","":"downstairs","session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3432025-02-28T06:07:34.019Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.130666401Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"downstairs","":"downstairs","session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3442025-02-28T06:07:34.019Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.130682291Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"downstairs","":"downstairs","session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3452025-02-28T06:07:34.019Z{"msg":"[1]R flush_numbers: [4000, 4000, 4000, 4000, 4000]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.130703341Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"downstairs","":"downstairs","session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3462025-02-28T06:07:34.019Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.130720162Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"downstairs","":"downstairs","session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3472025-02-28T06:07:34.019Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.130745072Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"downstairs","":"downstairs","session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3482025-02-28T06:07:34.019Z{"msg":"[2]R flush_numbers: [4000, 4000, 4000, 4000, 4000]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.130760182Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"downstairs","":"downstairs","session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3492025-02-28T06:07:34.019Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.130790833Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"downstairs","":"downstairs","session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3502025-02-28T06:07:34.020Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.130811384Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"downstairs","":"downstairs","session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3512025-02-28T06:07:34.020Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.130839734Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"downstairs","":"downstairs","session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3522025-02-28T06:07:34.020Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.130855744Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"downstairs","":"downstairs","session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3532025-02-28T06:07:34.020Z{"msg":"Next flush: 4001","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.130882895Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"downstairs","":"downstairs","session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3542025-02-28T06:07:34.020Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.130916706Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"downstairs","":"downstairs","session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3552025-02-28T06:07:34.020Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.130999007Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3562025-02-28T06:07:34.020Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.131065409Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"client":"0","":"downstairs","session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3572025-02-28T06:07:34.020Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.131198951Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"client":"1","":"downstairs","session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3582025-02-28T06:07:34.020Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.131267383Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"client":"2","":"downstairs","session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3592025-02-28T06:07:34.020Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.131297254Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3602025-02-28T06:07:34.020Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.131315954Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3612025-02-28T06:07:34.020Z{"msg":"345efba9-2a9c-428b-a9ca-28d076226af9 is now active with session: 376a641b-6047-4d0f-af59-33fe2f088a86","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.131336684Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3622025-02-28T06:07:34.020Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.131373905Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"session_id":"376a641b-6047-4d0f-af59-33fe2f088a86"}
3632025-02-28T06:07:34.020Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.131397336Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245}
3642025-02-28T06:07:34.020Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.131416006Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245}
3652025-02-28T06:07:34.020Z{"msg":"345efba9-2a9c-428b-a9ca-28d076226af9 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.131444717Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3662025-02-28T06:07:34.020Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.131458667Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"client":"0","":"downstairs","session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3672025-02-28T06:07:34.020Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.131477977Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"client":"0","":"downstairs","session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3682025-02-28T06:07:34.020Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.131532188Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"client":"1","":"downstairs","session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3692025-02-28T06:07:34.020Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.131547608Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"client":"1","":"downstairs","session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3702025-02-28T06:07:34.020Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.131564499Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"client":"2","":"downstairs","session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3712025-02-28T06:07:34.021Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.13160031Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"client":"2","":"downstairs","session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3722025-02-28T06:07:34.021Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.13162572Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"io task","client":"2","":"downstairs","session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3732025-02-28T06:07:34.021Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.131664241Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"io task","client":"0","":"downstairs","session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3742025-02-28T06:07:34.021Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.131698782Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"io task","client":"1","":"downstairs","session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3752025-02-28T06:07:34.021Z{"msg":"ds_connection connected from Ok(127.0.0.1:61738)","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.131751913Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"io task","client":"2","":"downstairs","session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3762025-02-28T06:07:34.021Z{"msg":"ds_connection connected from Ok(127.0.0.1:64973)","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.131781783Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"io task","client":"0","":"downstairs","session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3772025-02-28T06:07:34.021Z{"msg":"ds_connection connected from Ok(127.0.0.1:36580)","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.131805024Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"io task","client":"1","":"downstairs","session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3782025-02-28T06:07:34.021Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.131848795Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"client":"1","":"downstairs","session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3792025-02-28T06:07:34.021Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.131873045Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"client":"0","":"downstairs","session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3802025-02-28T06:07:34.021Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.131895016Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"client":"2","":"downstairs","session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3812025-02-28T06:07:34.021Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.13210432Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"client":"1","":"downstairs","session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3822025-02-28T06:07:34.021Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.132152561Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"client":"0","":"downstairs","session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3832025-02-28T06:07:34.022Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.132402616Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"client":"2","":"downstairs","session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3842025-02-28T06:07:34.022Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.132556549Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3852025-02-28T06:07:34.022Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.13259474Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3862025-02-28T06:07:34.022Z{"msg":"[0]R flush_numbers: [5322, 5322, 5322, 5322, 5322]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.132713163Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"downstairs","":"downstairs","session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3872025-02-28T06:07:34.022Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.132750973Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"downstairs","":"downstairs","session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3882025-02-28T06:07:34.022Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.132766894Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"downstairs","":"downstairs","session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3892025-02-28T06:07:34.022Z{"msg":"[1]R flush_numbers: [5322, 5322, 5322, 5322, 5322]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.132780384Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"downstairs","":"downstairs","session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3902025-02-28T06:07:34.022Z{"msg":"Wait for a query_work_queue command to finish before sending IO
3912025-02-28T06:07:34.022Z[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.132799704Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"downstairs","":"downstairs","session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3922025-02-28T06:07:34.022Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.132830905Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"downstairs","":"downstairs","session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3932025-02-28T06:07:34.023Z{"msg":"[2]R flush_numbers: [5322, 5322, 5322, 5322, 5322]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.132852815Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"downstairs","":"downstairs","session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3942025-02-28T06:07:34.023Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.132876516Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"downstairs","":"downstairs","session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3952025-02-28T06:07:34.023Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.132900047Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"downstairs","":"downstairs","session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3962025-02-28T06:07:34.023Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.132959628Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"downstairs","":"downstairs","session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3972025-02-28T06:07:34.023Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.132983508Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"downstairs","":"downstairs","session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3982025-02-28T06:07:34.023Z{"msg":"Next flush: 5323","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.133014649Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"downstairs","":"downstairs","session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
3992025-02-28T06:07:34.023Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.133034599Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"":"downstairs","":"downstairs","session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
4002025-02-28T06:07:34.023Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.133047679Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
4012025-02-28T06:07:34.023Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.13305966Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"client":"0","":"downstairs","session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
4022025-02-28T06:07:34.023Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.13307331Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"client":"1","":"downstairs","session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
4032025-02-28T06:07:34.023Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.133104051Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"client":"2","":"downstairs","session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
4042025-02-28T06:07:34.023Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.133137111Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
4052025-02-28T06:07:34.023Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.133160272Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
4062025-02-28T06:07:34.023Z{"msg":"345efba9-2a9c-428b-a9ca-28d076226af9 is now active with session: 5509d32f-246e-4eb1-b220-a1061237a9ee","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.133183512Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
4072025-02-28T06:07:34.023Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.133195842Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245,"session_id":"5509d32f-246e-4eb1-b220-a1061237a9ee"}
4082025-02-28T06:07:34.023Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:33.133225413Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245}
4092025-02-28T06:07:34.023ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
4102025-02-28T06:07:34.023ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4112025-02-28T06:07:34.023ZFill test
4122025-02-28T06:07:34.058ZRead and Verify all blocks (0..100 range:false)
4132025-02-28T06:07:34.137ZWrote out file "/var/tmp/test_up-build/verify_file"
4142025-02-28T06:07:34.137ZCLIENT: Tests done. All submitted work has been ACK'd
4152025-02-28T06:07:34.140Z----------------------------------------------------------------
4162025-02-28T06:07:34.140Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4172025-02-28T06:07:34.140ZStates: Active Active Active
4182025-02-28T06:07:34.140ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4192025-02-28T06:07:34.140Z 1002 Acked Read 50 Done Done Done false
4202025-02-28T06:07:34.140Z STATES DS:0 DS:1 DS:2 TOTAL
4212025-02-28T06:07:34.140Z Sent 0 0 0 0
4222025-02-28T06:07:34.140Z Done 1 1 1 3
4232025-02-28T06:07:34.140Z Skipped 0 0 0 0
4242025-02-28T06:07:34.140Z Error 0 0 0 0
4252025-02-28T06:07:34.140ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4262025-02-28T06:07:34.140ZDownstairs last five completed: 1001 1000
4272025-02-28T06:07:34.140ZUpstairs last five completed: 1002 1001 1000
4282025-02-28T06:07:34.140Z----------------------------------------------------------------
4292025-02-28T06:07:34.140Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4302025-02-28T06:07:34.140ZStates: Active Active Active
4312025-02-28T06:07:34.140ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4322025-02-28T06:07:34.141Z 1002 Acked Read 50 Sent Sent Done false
4332025-02-28T06:07:34.141Z STATES DS:0 DS:1 DS:2 TOTAL
4342025-02-28T06:07:34.141Z Sent 1 1 0 2
4352025-02-28T06:07:34.141Z Done 0 0 1 1
4362025-02-28T06:07:34.141Z Skipped 0 0 0 0
4372025-02-28T06:07:34.141Z Error 0 0 0 0
4382025-02-28T06:07:34.141ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4392025-02-28T06:07:34.141ZDownstairs last five completed: 1001 1000
4402025-02-28T06:07:34.141ZUpstairs last five completed: 1002 1001 1000
4412025-02-28T06:07:34.141ZCLIENT: Up:0 ds:2 act:6
4422025-02-28T06:07:38.139Z----------------------------------------------------------------
4432025-02-28T06:07:38.139Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4442025-02-28T06:07:38.139ZDownstairs last five completed: 1003 1002 1001 1000
4452025-02-28T06:07:38.139ZUpstairs last five completed: 1003 1002 1001 1000
4462025-02-28T06:07:38.139Z----------------------------------------------------------------
4472025-02-28T06:07:38.140Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4482025-02-28T06:07:38.140ZDownstairs last five completed: 1003 1002 1001 1000
4492025-02-28T06:07:38.140ZUpstairs last five completed: 1003 1002 1001 1000
4502025-02-28T06:07:38.143ZCLIENT: Up:0 ds:0 act:6
4512025-02-28T06:07:38.143ZCLIENT: All crucible jobs finished, exiting program
4522025-02-28T06:07:38.143Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-28T06:07:37.254815795Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1245}
4532025-02-28T06:07:38.148ZRepair part 1 passed
4542025-02-28T06:07:38.148Z
4552025-02-28T06:07:38.148ZKill the current downstairs
4562025-02-28T06:07:38.309ZDownstairs 2 stopped
4572025-02-28T06:07:38.310Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
4582025-02-28T06:07:38.313ZNow put back the original so we have a mismatch
4592025-02-28T06:07:38.313Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
4602025-02-28T06:07:38.315ZRestart downstairs with old directory
4612025-02-28T06:07:38.367Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4622025-02-28T06:07:38.384ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4632025-02-28T06:07:38.387Z 0 000-009 32 32 31 4001 4001 4000 F F F <---
4642025-02-28T06:07:38.387Z 1 010-019 32 32 31 4001 4001 4000 F F F <---
4652025-02-28T06:07:38.387Z 2 020-029 32 32 31 4001 4001 4000 F F F <---
4662025-02-28T06:07:38.387Z 3 030-039 32 32 31 4001 4001 4000 F F F <---
4672025-02-28T06:07:38.387Z 4 040-049 32 32 31 4001 4001 4000 F F F <---
4682025-02-28T06:07:38.387ZMax gen: 32, Max flush: 4001
4692025-02-28T06:07:38.387ZError: Difference in extent metadata found!
4702025-02-28T06:07:38.390Zdump test found error as expected
4712025-02-28T06:07:38.390Z
4722025-02-28T06:07:38.390Z
4732025-02-28T06:07:38.390Z/var/tmp/bins/crutest verify --range -g 33 -q --key 0zcV00IJ9kc5IAC8Oj/kAilIhUXe3EG2eWdkyXyg+TI= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
4742025-02-28T06:07:38.444Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.560551229Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"task":"crutest"}
4752025-02-28T06:07:38.447Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.561181052Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"task":"crutest"}
4762025-02-28T06:07:38.447Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.562776355Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"task":"crutest"}
4772025-02-28T06:07:38.447Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.562955978Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253}
4782025-02-28T06:07:38.447Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0c103f9ae2997264530dbd443fadbdf83878dac4\",\n git_commit_timestamp: \"2025-02-28T05:45:10.000000000Z\",\n git_branch: \"renovate/uuid-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.563004779Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253}
4792025-02-28T06:07:38.447Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.56303522Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253}
4802025-02-28T06:07:38.447Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.56305142Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253}
4812025-02-28T06:07:38.447Z{"msg":"Crucible 065645ee-3f0d-4e32-8971-88d0e1394094 has session id: 66530780-7462-4079-af0f-b992946c300c","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.563641273Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"session_id":"66530780-7462-4079-af0f-b992946c300c"}
4822025-02-28T06:07:38.447Z{"msg":"Upstairs opts: Upstairs UUID: 065645ee-3f0d-4e32-8971-88d0e1394094, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.563685984Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"session_id":"66530780-7462-4079-af0f-b992946c300c"}
4832025-02-28T06:07:38.448Z{"msg":"Crucible stats registered with UUID: 065645ee-3f0d-4e32-8971-88d0e1394094","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.563711914Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"session_id":"66530780-7462-4079-af0f-b992946c300c"}
4842025-02-28T06:07:38.448Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-28T06:07:37.563742445Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"session_id":"66530780-7462-4079-af0f-b992946c300c"}
4852025-02-28T06:07:38.451Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.565748356Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"task":"crutest"}
4862025-02-28T06:07:38.451Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.565829388Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253}
4872025-02-28T06:07:38.451Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0c103f9ae2997264530dbd443fadbdf83878dac4\",\n git_commit_timestamp: \"2025-02-28T05:45:10.000000000Z\",\n git_branch: \"renovate/uuid-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.565864129Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253}
4882025-02-28T06:07:38.452Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.565878519Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253}
4892025-02-28T06:07:38.452Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.565892199Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253}
4902025-02-28T06:07:38.452Z{"msg":"Crucible 065645ee-3f0d-4e32-8971-88d0e1394094 has session id: d4be74b8-38bc-4673-b3d0-a363824dce7d","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.566259937Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
4912025-02-28T06:07:38.452Z{"msg":"Upstairs opts: Upstairs UUID: 065645ee-3f0d-4e32-8971-88d0e1394094, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.566309198Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
4922025-02-28T06:07:38.452Z{"msg":"Crucible stats registered with UUID: 065645ee-3f0d-4e32-8971-88d0e1394094","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.566332878Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
4932025-02-28T06:07:38.452Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-28T06:07:37.566354168Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
4942025-02-28T06:07:38.452Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.566958771Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"task":"crutest"}
4952025-02-28T06:07:38.452Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.567053253Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253}
4962025-02-28T06:07:38.452Z{"msg":"065645ee-3f0d-4e32-8971-88d0e1394094 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.567072013Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"session_id":"66530780-7462-4079-af0f-b992946c300c"}
4972025-02-28T06:07:38.452Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.567086243Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"0","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
4982025-02-28T06:07:38.452Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.567123264Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"0","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
4992025-02-28T06:07:38.452Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.567145395Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"1","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5002025-02-28T06:07:38.452Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.567167105Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"io task","client":"0","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5012025-02-28T06:07:38.452Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.567190226Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"1","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5022025-02-28T06:07:38.452Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.567228777Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"2","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5032025-02-28T06:07:38.452Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.567253297Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"2","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5042025-02-28T06:07:38.452Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.567283558Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"io task","client":"2","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5052025-02-28T06:07:38.452Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.567357539Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"io task","client":"1","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5062025-02-28T06:07:38.452Z{"msg":"ds_connection connected from Ok(127.0.0.1:58320)","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.567509432Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"io task","client":"0","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5072025-02-28T06:07:38.452Z{"msg":"ds_connection connected from Ok(127.0.0.1:48807)","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.567548323Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"io task","client":"1","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5082025-02-28T06:07:38.452Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-28T06:07:37.567567284Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"io task","client":"2","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5092025-02-28T06:07:38.453Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-28T06:07:37.567604074Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"io task","client":"2","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5102025-02-28T06:07:38.453Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-28T06:07:37.567646365Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"session_id":"66530780-7462-4079-af0f-b992946c300c"}
5112025-02-28T06:07:38.453Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.567724177Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"io task","client":"2","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5122025-02-28T06:07:38.456Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.567966802Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"1","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5132025-02-28T06:07:38.456Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.568003602Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"0","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5142025-02-28T06:07:38.456Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.568613305Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"1","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5152025-02-28T06:07:38.456Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.568690307Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"0","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5162025-02-28T06:07:38.457Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.569048124Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"session_id":"66530780-7462-4079-af0f-b992946c300c"}
5172025-02-28T06:07:38.457Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:37.569083605Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"session_id":"66530780-7462-4079-af0f-b992946c300c"}
5182025-02-28T06:07:48.452Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.568537344Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"io task","client":"2","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5192025-02-28T06:07:48.452Z{"msg":"ds_connection connected from Ok(127.0.0.1:51756)","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.568609856Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"io task","client":"2","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5202025-02-28T06:07:48.457Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.56981707Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"2","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5212025-02-28T06:07:48.457Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.570375012Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"2","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5222025-02-28T06:07:48.457Z{"msg":"[0]R flush_numbers: [4001, 4001, 4001, 4001, 4001]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.570753Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"downstairs","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5232025-02-28T06:07:48.457Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.570785381Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"downstairs","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5242025-02-28T06:07:48.457Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.570802361Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"downstairs","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5252025-02-28T06:07:48.457Z{"msg":"[1]R flush_numbers: [4001, 4001, 4001, 4001, 4001]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.570816061Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"downstairs","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5262025-02-28T06:07:48.457Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.570837151Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"downstairs","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5272025-02-28T06:07:48.457Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.570851112Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"downstairs","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5282025-02-28T06:07:48.457Z{"msg":"[2]R flush_numbers: [4000, 4000, 4000, 4000, 4000]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.570870442Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"downstairs","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5292025-02-28T06:07:48.457Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.570883352Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"downstairs","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5302025-02-28T06:07:48.457Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.570903133Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"downstairs","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5312025-02-28T06:07:48.457Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.570916073Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"downstairs","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5322025-02-28T06:07:48.457Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.570935704Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"downstairs","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5332025-02-28T06:07:48.457Z{"msg":"Next flush: 4002","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.570948594Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"downstairs","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5342025-02-28T06:07:48.457Z{"msg":"extent 0 needs reconciliation: ExtentMetadata { gen: 32, flush: 4001, dirty: false } ExtentMetadata { gen: 32, flush: 4001, dirty: false } ExtentMetadata { gen: 31, flush: 4000, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.570971874Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"mend","":"downstairs","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5352025-02-28T06:07:48.457Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 4001, dirty: false }, ExtentMetadata { gen: 32, flush: 4001, dirty: false }, ExtentMetadata { gen: 31, flush: 4000, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.570988925Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"mend","":"downstairs","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5362025-02-28T06:07:48.457Z{"msg":"found dest for source 0 for extent at index 0 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.571017975Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"mend","":"downstairs","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5372025-02-28T06:07:48.457Z{"msg":"extent 1 needs reconciliation: ExtentMetadata { gen: 32, flush: 4001, dirty: false } ExtentMetadata { gen: 32, flush: 4001, dirty: false } ExtentMetadata { gen: 31, flush: 4000, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.571067796Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"mend","":"downstairs","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5382025-02-28T06:07:48.457Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 4001, dirty: false }, ExtentMetadata { gen: 32, flush: 4001, dirty: false }, ExtentMetadata { gen: 31, flush: 4000, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.571093787Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"mend","":"downstairs","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5392025-02-28T06:07:48.458Z{"msg":"found dest for source 0 for extent at index 1 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.571114797Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"mend","":"downstairs","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5402025-02-28T06:07:48.458Z{"msg":"extent 2 needs reconciliation: ExtentMetadata { gen: 32, flush: 4001, dirty: false } ExtentMetadata { gen: 32, flush: 4001, dirty: false } ExtentMetadata { gen: 31, flush: 4000, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.571130828Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"mend","":"downstairs","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5412025-02-28T06:07:48.458Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 4001, dirty: false }, ExtentMetadata { gen: 32, flush: 4001, dirty: false }, ExtentMetadata { gen: 31, flush: 4000, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.571146638Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"mend","":"downstairs","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5422025-02-28T06:07:48.458Z{"msg":"found dest for source 0 for extent at index 2 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.571160328Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"mend","":"downstairs","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5432025-02-28T06:07:48.458Z{"msg":"extent 3 needs reconciliation: ExtentMetadata { gen: 32, flush: 4001, dirty: false } ExtentMetadata { gen: 32, flush: 4001, dirty: false } ExtentMetadata { gen: 31, flush: 4000, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.571175929Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"mend","":"downstairs","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5442025-02-28T06:07:48.458Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 4001, dirty: false }, ExtentMetadata { gen: 32, flush: 4001, dirty: false }, ExtentMetadata { gen: 31, flush: 4000, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.571197679Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"mend","":"downstairs","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5452025-02-28T06:07:48.458Z{"msg":"found dest for source 0 for extent at index 3 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.571211329Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"mend","":"downstairs","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5462025-02-28T06:07:48.460Z{"msg":"extent 4 needs reconciliation: ExtentMetadata { gen: 32, flush: 4001, dirty: false } ExtentMetadata { gen: 32, flush: 4001, dirty: false } ExtentMetadata { gen: 31, flush: 4000, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.57123291Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"mend","":"downstairs","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5472025-02-28T06:07:48.460Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 4001, dirty: false }, ExtentMetadata { gen: 32, flush: 4001, dirty: false }, ExtentMetadata { gen: 31, flush: 4000, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.57124807Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"mend","":"downstairs","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5482025-02-28T06:07:48.460Z{"msg":"found dest for source 0 for extent at index 4 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.57126795Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"mend","":"downstairs","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5492025-02-28T06:07:48.460Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.571291531Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"0","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5502025-02-28T06:07:48.460Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.571313861Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"1","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5512025-02-28T06:07:48.460Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.571331882Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"2","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5522025-02-28T06:07:48.460Z{"msg":"Full repair list: {ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.571366522Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"downstairs","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5532025-02-28T06:07:48.460Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 4002, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 4002, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 4002, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 4002, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 4002, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.571466815Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"downstairs","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5542025-02-28T06:07:48.462Z{"msg":"starting reconciliation 64c9880b-e359-4f22-833b-d9ecb99b8a5f: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.571505235Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"downstairs","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5552025-02-28T06:07:48.462Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.571687049Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"2","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5562025-02-28T06:07:48.467Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.580605312Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"2","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5572025-02-28T06:07:48.470Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.584163576Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"2","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5582025-02-28T06:07:48.473Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.58775533Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"2","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5592025-02-28T06:07:48.475Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.591346164Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"2","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5602025-02-28T06:07:48.478Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.594503879Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"downstairs","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5612025-02-28T06:07:48.478Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.594540859Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"0","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5622025-02-28T06:07:48.479Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.59455787Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"1","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5632025-02-28T06:07:48.479Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.59457177Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"2","":"downstairs","session_id":"66530780-7462-4079-af0f-b992946c300c"}
5642025-02-28T06:07:48.479Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.59458519Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"session_id":"66530780-7462-4079-af0f-b992946c300c"}
5652025-02-28T06:07:48.479Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.594597791Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"session_id":"66530780-7462-4079-af0f-b992946c300c"}
5662025-02-28T06:07:48.479Z{"msg":"065645ee-3f0d-4e32-8971-88d0e1394094 is now active with session: 66530780-7462-4079-af0f-b992946c300c","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.594612091Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"session_id":"66530780-7462-4079-af0f-b992946c300c"}
5672025-02-28T06:07:48.479Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.594637732Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253}
5682025-02-28T06:07:48.479Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.594729463Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253}
5692025-02-28T06:07:48.479Z{"msg":"065645ee-3f0d-4e32-8971-88d0e1394094 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.594754354Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
5702025-02-28T06:07:48.479Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.594767754Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"0","":"downstairs","session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
5712025-02-28T06:07:48.479Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.594789895Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"0","":"downstairs","session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
5722025-02-28T06:07:48.480Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.594803575Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"1","":"downstairs","session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
5732025-02-28T06:07:48.480Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.594824845Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"1","":"downstairs","session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
5742025-02-28T06:07:48.480Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.594849706Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"2","":"downstairs","session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
5752025-02-28T06:07:48.483Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.594877486Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"2","":"downstairs","session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
5762025-02-28T06:07:48.483Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.594916987Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"io task","client":"2","":"downstairs","session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
5772025-02-28T06:07:48.483Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.594944508Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"io task","client":"0","":"downstairs","session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
5782025-02-28T06:07:48.483Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.594972278Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"io task","client":"1","":"downstairs","session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
5792025-02-28T06:07:48.483Z{"msg":"ds_connection connected from Ok(127.0.0.1:35325)","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.594999129Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"io task","client":"2","":"downstairs","session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
5802025-02-28T06:07:48.483Z{"msg":"ds_connection connected from Ok(127.0.0.1:35673)","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.595025289Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"io task","client":"1","":"downstairs","session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
5812025-02-28T06:07:48.483Z{"msg":"ds_connection connected from Ok(127.0.0.1:40293)","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.59504934Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"io task","client":"0","":"downstairs","session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
5822025-02-28T06:07:48.483Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.595440698Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"0","":"downstairs","session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
5832025-02-28T06:07:48.483Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.595567211Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"1","":"downstairs","session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
5842025-02-28T06:07:48.483Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.595608312Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"2","":"downstairs","session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
5852025-02-28T06:07:48.483Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.595971019Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"0","":"downstairs","session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
5862025-02-28T06:07:48.483Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.596110482Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"2","":"downstairs","session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
5872025-02-28T06:07:48.483Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.596141663Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"1","":"downstairs","session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
5882025-02-28T06:07:48.483Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.596381947Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
5892025-02-28T06:07:48.483Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.59649939Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
5902025-02-28T06:07:48.483Z{"msg":"[0]R flush_numbers: [5323, 5323, 5323, 5323, 5323]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.59653396Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"downstairs","":"downstairs","session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
5912025-02-28T06:07:48.484Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.596549531Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"downstairs","":"downstairs","session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
5922025-02-28T06:07:48.484Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.596573871Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"downstairs","":"downstairs","session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
5932025-02-28T06:07:48.484Z{"msg":"[1]R flush_numbers: [5323, 5323, 5323, 5323, 5323]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.596590852Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"downstairs","":"downstairs","session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
5942025-02-28T06:07:48.484Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.596616672Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"downstairs","":"downstairs","session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
5952025-02-28T06:07:48.484Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.596631673Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"downstairs","":"downstairs","session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
5962025-02-28T06:07:48.484Z{"msg":"Wait for a query_work_queue command to finish before sending IO
5972025-02-28T06:07:48.484Z[2]R flush_numbers: [5323, 5323, 5323, 5323, 5323]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.596659903Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"downstairs","":"downstairs","session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
5982025-02-28T06:07:48.484Z{"msg":"[2]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.596675094Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"downstairs","":"downstairs","session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
5992025-02-28T06:07:48.484Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.596688674Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"downstairs","":"downstairs","session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
6002025-02-28T06:07:48.484Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.596701234Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"downstairs","":"downstairs","session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
6012025-02-28T06:07:48.484Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.596722624Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"downstairs","":"downstairs","session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
6022025-02-28T06:07:48.484Z{"msg":"Next flush: 5324","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.596735815Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"downstairs","":"downstairs","session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
6032025-02-28T06:07:48.484Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.596749575Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"":"downstairs","":"downstairs","session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
6042025-02-28T06:07:48.484Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.596770015Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
6052025-02-28T06:07:48.484Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.596782376Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"0","":"downstairs","session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
6062025-02-28T06:07:48.484Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.596795066Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"1","":"downstairs","session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
6072025-02-28T06:07:48.484Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.596820357Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"client":"2","":"downstairs","session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
6082025-02-28T06:07:48.484Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.596834817Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
6092025-02-28T06:07:48.484Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.596857007Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
6102025-02-28T06:07:48.484Z{"msg":"065645ee-3f0d-4e32-8971-88d0e1394094 is now active with session: d4be74b8-38bc-4673-b3d0-a363824dce7d","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.596878858Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
6112025-02-28T06:07:48.484Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.596908318Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253,"session_id":"d4be74b8-38bc-4673-b3d0-a363824dce7d"}
6122025-02-28T06:07:48.484Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-02-28T06:07:47.596929749Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253}
6132025-02-28T06:07:48.484ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
6142025-02-28T06:07:48.484ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
6152025-02-28T06:07:48.484ZRead and Verify all blocks (0..100 range:true)
6162025-02-28T06:07:48.560ZWrote out file "/var/tmp/test_up-build/verify_file"
6172025-02-28T06:07:48.560ZVerify test completed
6182025-02-28T06:07:48.563ZWrote out file "/var/tmp/test_up-build/verify_file"
6192025-02-28T06:07:48.563ZCLIENT: Tests done. All submitted work has been ACK'd
6202025-02-28T06:07:48.564Z----------------------------------------------------------------
6212025-02-28T06:07:48.564Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6222025-02-28T06:07:48.564ZStates: Active Active Active
6232025-02-28T06:07:48.564ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6242025-02-28T06:07:48.564Z 1000 Acked Read 50 Done Done Done false
6252025-02-28T06:07:48.564Z STATES DS:0 DS:1 DS:2 TOTAL
6262025-02-28T06:07:48.564Z Sent 0 0 0 0
6272025-02-28T06:07:48.564Z Done 1 1 1 3
6282025-02-28T06:07:48.564Z Skipped 0 0 0 0
6292025-02-28T06:07:48.564Z Error 0 0 0 0
6302025-02-28T06:07:48.564ZLast Flush: None None None
6312025-02-28T06:07:48.564ZDownstairs last five completed:
6322025-02-28T06:07:48.564ZUpstairs last five completed: 1000
6332025-02-28T06:07:48.564Z----------------------------------------------------------------
6342025-02-28T06:07:48.564Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6352025-02-28T06:07:48.564ZStates: Active Active Active
6362025-02-28T06:07:48.564ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6372025-02-28T06:07:48.564Z 1000 Acked Read 50 Done Done Done false
6382025-02-28T06:07:48.564Z STATES DS:0 DS:1 DS:2 TOTAL
6392025-02-28T06:07:48.564Z Sent 0 0 0 0
6402025-02-28T06:07:48.564Z Done 1 1 1 3
6412025-02-28T06:07:48.564Z Skipped 0 0 0 0
6422025-02-28T06:07:48.564Z Error 0 0 0 0
6432025-02-28T06:07:48.564ZLast Flush: None None None
6442025-02-28T06:07:48.564ZDownstairs last five completed:
6452025-02-28T06:07:48.564ZUpstairs last five completed: 1000
6462025-02-28T06:07:48.564ZCLIENT: Up:0 ds:2 act:6
6472025-02-28T06:07:52.563Z----------------------------------------------------------------
6482025-02-28T06:07:52.563Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6492025-02-28T06:07:52.563ZDownstairs last five completed: 1001 1000
6502025-02-28T06:07:52.563ZUpstairs last five completed: 1001 1000
6512025-02-28T06:07:52.566Z----------------------------------------------------------------
6522025-02-28T06:07:52.566Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6532025-02-28T06:07:52.566ZDownstairs last five completed: 1001 1000
6542025-02-28T06:07:52.566ZUpstairs last five completed: 1001 1000
6552025-02-28T06:07:52.566ZCLIENT: Up:0 ds:0 act:6
6562025-02-28T06:07:52.566ZCLIENT: All crucible jobs finished, exiting program
6572025-02-28T06:07:52.566Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-28T06:07:51.679254462Z","hostname":"w-01JN5J6KH8PE1MH4X3YMYEWZDH","pid":1253}
6582025-02-28T06:07:52.570ZRepair part 2 passed
6592025-02-28T06:07:52.572Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
6602025-02-28T06:07:52.588ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6612025-02-28T06:07:52.588Z 0 000-009 32 32 32 4001 4001 4001 F F F
6622025-02-28T06:07:52.590Z 1 010-019 32 32 32 4001 4001 4001 F F F
6632025-02-28T06:07:52.590Z 2 020-029 32 32 32 4001 4001 4001 F F F
6642025-02-28T06:07:52.590Z 3 030-039 32 32 32 4001 4001 4001 F F F
6652025-02-28T06:07:52.591Z 4 040-049 32 32 32 4001 4001 4001 F F F
6662025-02-28T06:07:52.591ZMax gen: 32, Max flush: 4001
6672025-02-28T06:07:52.593Zdump test passed
6682025-02-28T06:07:52.593Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
6692025-02-28T06:07:52.608Z Extent 1
6702025-02-28T06:07:52.611ZGEN 32 32 32
6712025-02-28T06:07:52.611ZFLUSH_ID 4001 4001 4001
6722025-02-28T06:07:52.611ZDIRTY
6732025-02-28T06:07:52.611Z
6742025-02-28T06:07:52.611ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
6752025-02-28T06:07:52.611Z010 A A A A A A
6762025-02-28T06:07:52.613Z011 A A A A A A
6772025-02-28T06:07:52.613Z012 A A A A A A
6782025-02-28T06:07:52.616Z013 A A A A A A
6792025-02-28T06:07:52.616Z014 A A A A A A
6802025-02-28T06:07:52.619Z015 A A A A A A
6812025-02-28T06:07:52.619Z016 A A A A A A
6822025-02-28T06:07:52.621Z017 A A A A A A
6832025-02-28T06:07:52.624Z018 A A A A A A
6842025-02-28T06:07:52.624Z019 A A A A A A
6852025-02-28T06:07:52.626Zdump extent test passed
6862025-02-28T06:07:52.629Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
6872025-02-28T06:07:52.643ZExtent 2 Block in extent 0 Actual block 20
6882025-02-28T06:07:52.645Z
6892025-02-28T06:07:52.645Z DATA SHA256 VER
6902025-02-28T06:07:52.645Z------ ---------------------------------------------------------------- ---
6912025-02-28T06:07:52.645Z 0 826690ecd5d863f4e80fd1b19f5d17552feba76a6e81b787a6f3bcd26964ad08 A
6922025-02-28T06:07:52.648Z 1 826690ecd5d863f4e80fd1b19f5d17552feba76a6e81b787a6f3bcd26964ad08 A
6932025-02-28T06:07:52.648Z 2 826690ecd5d863f4e80fd1b19f5d17552feba76a6e81b787a6f3bcd26964ad08 A
6942025-02-28T06:07:52.648Z
6952025-02-28T06:07:52.648ZNONCES 0 1 2 DIFF
6962025-02-28T06:07:52.648Z------ ------------------------ ------------------------ ------------------------ -----
6972025-02-28T06:07:52.648Z 0 f23f44fc71da3af23249fd25 f23f44fc71da3af23249fd25 f23f44fc71da3af23249fd25
6982025-02-28T06:07:52.648Z
6992025-02-28T06:07:52.648Z TAGS 0 1 2 DIFF
7002025-02-28T06:07:52.648Z------ -------------------------------- -------------------------------- -------------------------------- -----
7012025-02-28T06:07:52.648Z 0 ba89e4ff6d40bb132417ec4e8f1488b0 ba89e4ff6d40bb132417ec4e8f1488b0 ba89e4ff6d40bb132417ec4e8f1488b0
7022025-02-28T06:07:52.649Z
7032025-02-28T06:07:52.649ZHASHES 0 1 2 DIFF
7042025-02-28T06:07:52.649Z------ ---------------- ---------------- ---------------- -----
7052025-02-28T06:07:52.649Z
7062025-02-28T06:07:52.652Zdump block test passed
7072025-02-28T06:07:52.652ZInitial upstairs tests have completed, stopping all downstairs
7082025-02-28T06:07:57.715ZCreating 7 larger downstairs regions
7092025-02-28T06:07:57.899ZStarting 7 downstairs
7102025-02-28T06:08:02.906Zdsc restarted at PID: 1269
7112025-02-28T06:08:03.291ZNow do the replace-reconcile test
7122025-02-28T06:08:03.345ZUsing 8870 for the replacement port
7132025-02-28T06:10:56.549ZCompleted test: replace-reconcile
7142025-02-28T06:10:56.553ZNow do the replace-before-active test
7152025-02-28T06:14:30.233ZCompleted test: replace-before-active
7162025-02-28T06:14:30.233ZAll tests have completed, stopping all downstairs
7172025-02-28T06:14:30.294Z
7182025-02-28T06:14:30.294ZAll Tests have passed
7192025-02-28T06:14:30.298Z13:34 Test duration
7202025-02-28T06:14:30.298Z
7212025-02-28T06:14:30.298Zreal 13:33.517207622
7222025-02-28T06:14:30.298Zuser 18:34.332090285
7232025-02-28T06:14:30.298Zsys 24.602501926
7242025-02-28T06:14:30.298Ztrap 0.530559478
7252025-02-28T06:14:30.298Ztflt 0.034061524
7262025-02-28T06:14:30.298Zdflt 0.038104490
7272025-02-28T06:14:30.298Zkflt 0.001162839
7282025-02-28T06:14:30.298Zlock 25:20:10.279949407
7292025-02-28T06:14:30.298Zslp 1:14:04.861139406
7302025-02-28T06:14:30.299Zlat 1:59.261024123
7312025-02-28T06:14:30.299Zstop 0.205322532
7322025-02-28T06:14:30.299Z+ echo 'test-up-2region-encrypted ends'
7332025-02-28T06:14:30.299Ztest-up-2region-encrypted ends
7342025-02-28T06:14:35.300Zprocess exited: duration 821732 ms, exit code 0
7352025-02-28T06:14:35.300Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
7362025-02-28T06:15:35.341Zexec warning: : stdout descriptor may be held open by a background process; giving up!
7372025-02-28T06:15:35.341Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
7382025-02-28T06:15:35.354Zfound 17 output files
7392025-02-28T06:15:35.354Zuploading: /tmp/test_up-build/dsc-out.txt (50920 bytes)
7402025-02-28T06:15:36.367Zuploaded: /tmp/test_up-build/dsc-out.txt
7412025-02-28T06:15:36.376Zuploading: /tmp/test_up-build/test_up_out.txt (8124571 bytes)
7422025-02-28T06:15:37.464Zuploaded: /tmp/test_up-build/test_up_out.txt
7432025-02-28T06:15:37.464Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (147267 bytes)
7442025-02-28T06:15:38.481Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
7452025-02-28T06:15:38.484Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (144639 bytes)
7462025-02-28T06:15:39.499Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
7472025-02-28T06:15:39.500Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (250974 bytes)
7482025-02-28T06:15:40.521Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
7492025-02-28T06:15:40.523Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (139426 bytes)
7502025-02-28T06:15:41.539Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
7512025-02-28T06:15:41.539Zuploading: /tmp/test_up-build/dsc/downstairs-8850.txt (2502 bytes)
7522025-02-28T06:15:42.547Zuploaded: /tmp/test_up-build/dsc/downstairs-8850.txt
7532025-02-28T06:15:42.547Zuploading: /tmp/test_up-build/dsc/downstairs-8860.txt (33791 bytes)
7542025-02-28T06:15:43.558Zuploaded: /tmp/test_up-build/dsc/downstairs-8860.txt
7552025-02-28T06:15:43.558Zuploading: /tmp/test_up-build/dsc/downstairs-8870.txt (841865 bytes)
7562025-02-28T06:15:44.596Zuploaded: /tmp/test_up-build/dsc/downstairs-8870.txt
7572025-02-28T06:15:44.596Zuploading: /tmp/debug/df.txt (1270 bytes)
7582025-02-28T06:15:45.605Zuploaded: /tmp/debug/df.txt
7592025-02-28T06:15:45.605Zuploading: /tmp/debug/dtrace.txt (961835 bytes)
7602025-02-28T06:15:46.647Zuploaded: /tmp/debug/dtrace.txt
7612025-02-28T06:15:46.650Zuploading: /tmp/debug/iostat.txt (264649 bytes)
7622025-02-28T06:15:46.665Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 264649 -> 267960
7632025-02-28T06:15:47.672Zuploaded: /tmp/debug/iostat.txt
7642025-02-28T06:15:47.675Zuploading: /tmp/debug/mpstat.txt (641685 bytes)
7652025-02-28T06:15:47.704Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 641685 -> 650445
7662025-02-28T06:15:48.710Zuploaded: /tmp/debug/mpstat.txt
7672025-02-28T06:15:48.713Zuploading: /tmp/debug/paging.txt (115868 bytes)
7682025-02-28T06:15:48.722Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 115868 -> 117692
7692025-02-28T06:15:49.728Zuploaded: /tmp/debug/paging.txt
7702025-02-28T06:15:49.728Zuploading: /tmp/debug/prstat.txt (1247997 bytes)
7712025-02-28T06:15:49.771Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 1247997 -> 1266116
7722025-02-28T06:15:50.776Zuploaded: /tmp/debug/prstat.txt
7732025-02-28T06:15:50.776Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
7742025-02-28T06:15:51.785Zuploaded: /tmp/debug/psrinfo.txt
7752025-02-28T06:15:51.785Zuploading: /tmp/debug/upstairs-info.txt (180070 bytes)
7762025-02-28T06:15:52.804Zuploaded: /tmp/debug/upstairs-info.txt