01JNRW1T2J7DWG7W3CTQM6867D: test-up-2region-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01JNRW28FZY4JHX739KF62T00Y

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12025-03-07T18:11:05.718Zjob dependencies complete; ready to run (waiting for 13 m 48 s)
22025-03-07T18:12:02.975Zjob assigned to worker 01JNRWVM07N8XV2TSYHKMREPCJ [factory aws, i-054c16bce9892cd2e] (queued for 57 s)
32025-03-07T18:12:11.064Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42025-03-07T18:12:12.281Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52025-03-07T18:12:12.281Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62025-03-07T18:12:13.353Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72025-03-07T18:12:13.353Zdownloading input: /input/build/work/bins/crutest.gz
82025-03-07T18:12:14.658Zdownloaded input: /input/build/work/bins/crutest.gz
92025-03-07T18:12:14.658Zdownloading input: /input/build/work/bins/dsc.gz
102025-03-07T18:12:15.479Zdownloaded input: /input/build/work/bins/dsc.gz
112025-03-07T18:12:15.484Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122025-03-07T18:12:15.497Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132025-03-07T18:12:15.497Zdownloading input: /input/build/work/scripts/test_ds.sh
142025-03-07T18:12:15.512Zdownloaded input: /input/build/work/scripts/test_ds.sh
152025-03-07T18:12:15.512Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162025-03-07T18:12:15.532Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172025-03-07T18:12:15.532Zdownloading input: /input/build/work/scripts/test_repair.sh
182025-03-07T18:12:15.552Zdownloaded input: /input/build/work/scripts/test_repair.sh
192025-03-07T18:12:15.552Zdownloading input: /input/build/work/scripts/test_replay.sh
202025-03-07T18:12:15.571Zdownloaded input: /input/build/work/scripts/test_replay.sh
212025-03-07T18:12:15.571Zdownloading input: /input/build/work/scripts/test_up.sh
222025-03-07T18:12:15.593Zdownloaded input: /input/build/work/scripts/test_up.sh
232025-03-07T18:12:15.593Zdownloading input: /input/build/work/scripts/upstairs_info.d
242025-03-07T18:12:15.617Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252025-03-07T18:12:15.621Zdownloading input: /input/build/tmp/cargo-test-out.log
262025-03-07T18:12:16.333Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272025-03-07T18:12:16.333Zstarting task 0: "setup"
282025-03-07T18:12:16.339Z++ uname -s
292025-03-07T18:12:16.343Z+ kern=SunOS
302025-03-07T18:12:16.343Z+ build_user=build
312025-03-07T18:12:16.344Z+ build_uid=12345
322025-03-07T18:12:16.344Z+ work_dir=/work
332025-03-07T18:12:16.344Z+ input_dir=/input
342025-03-07T18:12:16.344Z+ [[ 0 == 12345 ]]
352025-03-07T18:12:16.344Z+ case "$kern" in
362025-03-07T18:12:16.344Z+ groupadd -g 12345 build
372025-03-07T18:12:16.348Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382025-03-07T18:12:18.353Z+ zfs create -o mountpoint=/work rpool/work
392025-03-07T18:12:18.537Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402025-03-07T18:12:18.542Z+ home_fs=zfs
412025-03-07T18:12:18.542Z+ [[ zfs == autofs ]]
422025-03-07T18:12:18.542Z+ mkdir -p /home/build
432025-03-07T18:12:18.546Z+ chown build:build /home/build /work
442025-03-07T18:12:20.543Z+ chmod 0700 /home/build /work
452025-03-07T18:12:20.547Zprocess exited: duration 4217 ms, exit code 0
 
462025-03-07T18:12:20.555Zstarting task 1: "authentication"
472025-03-07T18:12:20.573Zprocess exited: duration 18 ms, exit code 0
 
482025-03-07T18:12:20.581Zstarting task 2: "build"
492025-03-07T18:12:20.586Z+ banner cores
502025-03-07T18:12:20.589Z
512025-03-07T18:12:20.590Z #### #### ##### ###### ####
522025-03-07T18:12:20.590Z # # # # # # # #
532025-03-07T18:12:20.590Z # # # # # ##### ####
542025-03-07T18:12:20.590Z # # # ##### # #
552025-03-07T18:12:20.590Z # # # # # # # # #
562025-03-07T18:12:20.590Z #### #### # # ###### ####
572025-03-07T18:12:20.590Z
582025-03-07T18:12:20.590Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
592025-03-07T18:12:20.598Z+ banner unpack
602025-03-07T18:12:20.613Z
612025-03-07T18:12:20.613Z # # # # ##### ## #### # #
622025-03-07T18:12:20.613Z # # ## # # # # # # # # #
632025-03-07T18:12:20.613Z # # # # # # # # # # ####
642025-03-07T18:12:20.613Z # # # # # ##### ###### # # #
652025-03-07T18:12:20.613Z # # # ## # # # # # # #
662025-03-07T18:12:20.613Z #### # # # # # #### # #
672025-03-07T18:12:20.613Z
682025-03-07T18:12:20.613Z+ mkdir -p /var/tmp/bins
692025-03-07T18:12:20.613Z+ for t in "$input/bins/"*.gz
702025-03-07T18:12:20.617Z++ basename /input/build/work/bins/crucible-downstairs.gz
712025-03-07T18:12:20.617Z+ b=crucible-downstairs.gz
722025-03-07T18:12:20.617Z+ b=crucible-downstairs
732025-03-07T18:12:20.617Z+ gunzip
742025-03-07T18:12:23.086Z+ chmod +x /var/tmp/bins/crucible-downstairs
752025-03-07T18:12:23.089Z+ for t in "$input/bins/"*.gz
762025-03-07T18:12:23.090Z++ basename /input/build/work/bins/crucible-hammer.gz
772025-03-07T18:12:23.090Z+ b=crucible-hammer.gz
782025-03-07T18:12:23.090Z+ b=crucible-hammer
792025-03-07T18:12:23.090Z+ gunzip
802025-03-07T18:12:25.253Z+ chmod +x /var/tmp/bins/crucible-hammer
812025-03-07T18:12:25.256Z+ for t in "$input/bins/"*.gz
822025-03-07T18:12:25.256Z++ basename /input/build/work/bins/crutest.gz
832025-03-07T18:12:25.259Z+ b=crutest.gz
842025-03-07T18:12:25.259Z+ b=crutest
852025-03-07T18:12:25.259Z+ gunzip
862025-03-07T18:12:27.692Z+ chmod +x /var/tmp/bins/crutest
872025-03-07T18:12:27.695Z+ for t in "$input/bins/"*.gz
882025-03-07T18:12:27.696Z++ basename /input/build/work/bins/dsc.gz
892025-03-07T18:12:27.699Z+ b=dsc.gz
902025-03-07T18:12:27.699Z+ b=dsc
912025-03-07T18:12:27.699Z+ gunzip
922025-03-07T18:12:29.030Z+ chmod +x /var/tmp/bins/dsc
932025-03-07T18:12:29.033Z+ export BINDIR=/var/tmp/bins
942025-03-07T18:12:29.033Z+ BINDIR=/var/tmp/bins
952025-03-07T18:12:29.033Z+ jobpid=1146
962025-03-07T18:12:29.033Z+ echo 'Setup debug logging'
972025-03-07T18:12:29.036ZSetup debug logging
982025-03-07T18:12:29.036Z+ mkdir /tmp/debug
992025-03-07T18:12:29.036Z+ sleep 3600
1002025-03-07T18:12:29.036Z+ psrinfo -v
1012025-03-07T18:12:29.040Z+ df -h
1022025-03-07T18:12:29.046Z+ prstat -d d -mLc 1
1032025-03-07T18:12:29.049Z+ iostat -T d -xn 1
1042025-03-07T18:12:29.049Z+ mpstat -T d 1
1052025-03-07T18:12:29.049Z+ vmstat -T d -p 1
1062025-03-07T18:12:29.052Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1072025-03-07T18:12:29.053Z+ banner test_up_2r_encrypted
1082025-03-07T18:12:29.053Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1092025-03-07T18:12:29.053Z #####
1102025-03-07T18:12:29.053Z ##### ###### #### ##### # # ##### # # #####
1112025-03-07T18:12:29.053Z # # # # # # # # # # #
1122025-03-07T18:12:29.053Z # ##### #### # # # # # ##### # #
1132025-03-07T18:12:29.053Z # # # # # # ##### # #####
1142025-03-07T18:12:29.053Z # # # # # # # # # # #
1152025-03-07T18:12:29.053Z # ###### #### # ####### #### # ####### ####### # #
1162025-03-07T18:12:29.053Z
1172025-03-07T18:12:29.053Z+ ptime -m bash /input/build/work/scripts/test_up.sh -r 2 -N encrypted
1182025-03-07T18:12:29.064Z/input/build/work
1192025-03-07T18:12:29.078ZUsing 2 region sets
1202025-03-07T18:12:29.078ZTurn off color for downstairs dump
1212025-03-07T18:12:29.095ZUpstairs using key: JBkvRCjQCWaFPGAUVme4+TRlliNFHE34UEl9Bakpqto=
1222025-03-07T18:12:29.099Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1232025-03-07T18:12:29.099ZCreating 6 downstairs regions
1242025-03-07T18:12:29.448ZStarting 6 downstairs
1252025-03-07T18:12:29.452Zdsc started at PID: 1194
1262025-03-07T18:12:34.783ZDisable automatic restart on all downstairs
1272025-03-07T18:12:34.837Z
1282025-03-07T18:12:34.837ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1292025-03-07T18:12:34.841ZRunning test: span
1302025-03-07T18:12:39.011ZCompleted test: span
1312025-03-07T18:12:44.017ZRunning test: big
1322025-03-07T18:12:48.599ZCompleted test: big
1332025-03-07T18:12:53.604ZRunning test: dep
1342025-03-07T18:13:27.028ZCompleted test: dep
1352025-03-07T18:13:32.034ZRunning test: balloon
1362025-03-07T18:17:47.970ZCompleted test: balloon
1372025-03-07T18:17:52.693ZRunning test: deactivate
1382025-03-07T18:18:47.386ZCompleted test: deactivate
1392025-03-07T18:18:52.393ZRunning hammer
1402025-03-07T18:19:05.205ZRun repair tests
1412025-03-07T18:19:05.208Z/var/tmp/bins/crutest fill -g 31 -q --key JBkvRCjQCWaFPGAUVme4+TRlliNFHE34UEl9Bakpqto= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1422025-03-07T18:19:05.267Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.188952151Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"task":"crutest"}
1432025-03-07T18:19:05.270Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.189475758Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"task":"crutest"}
1442025-03-07T18:19:05.270Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.191151584Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"task":"crutest"}
1452025-03-07T18:19:05.270Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.191334425Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237}
1462025-03-07T18:19:05.270Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.19137838Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237}
1472025-03-07T18:19:05.270Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.191401928Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237}
1482025-03-07T18:19:05.270Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.191418123Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237}
1492025-03-07T18:19:05.270Z{"msg":"Crucible 8bee2d88-2c7d-45a0-a52a-39646ec755e3 has session id: ca73f76b-eb90-4d9f-92cb-56e78ffffca5","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.192005222Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1502025-03-07T18:19:05.270Z{"msg":"Upstairs opts: Upstairs UUID: 8bee2d88-2c7d-45a0-a52a-39646ec755e3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.192053178Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1512025-03-07T18:19:05.270Z{"msg":"Crucible stats registered with UUID: 8bee2d88-2c7d-45a0-a52a-39646ec755e3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.192077306Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1522025-03-07T18:19:05.270Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:04.192116499Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1532025-03-07T18:19:05.274Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.194049232Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"task":"crutest"}
1542025-03-07T18:19:05.274Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.194116484Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237}
1552025-03-07T18:19:05.274Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.19416218Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237}
1562025-03-07T18:19:05.274Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.194177225Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237}
1572025-03-07T18:19:05.274Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.19419211Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237}
1582025-03-07T18:19:05.274Z{"msg":"Crucible 8bee2d88-2c7d-45a0-a52a-39646ec755e3 has session id: 36a45bf0-49af-4d67-93a1-55d3287444e2","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.194593915Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
1592025-03-07T18:19:05.274Z{"msg":"Upstairs opts: Upstairs UUID: 8bee2d88-2c7d-45a0-a52a-39646ec755e3, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.194651025Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
1602025-03-07T18:19:05.274Z{"msg":"Crucible stats registered with UUID: 8bee2d88-2c7d-45a0-a52a-39646ec755e3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.194675263Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
1612025-03-07T18:19:05.274Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:04.19469637Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
1622025-03-07T18:19:05.274Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.195275926Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"task":"crutest"}
1632025-03-07T18:19:05.274Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.195306506Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237}
1642025-03-07T18:19:05.274Z{"msg":"8bee2d88-2c7d-45a0-a52a-39646ec755e3 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.195402518Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1652025-03-07T18:19:05.274Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.195431278Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"client":"0","":"downstairs","session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1662025-03-07T18:19:05.274Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.195538204Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"client":"0","":"downstairs","session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1672025-03-07T18:19:05.274Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.19558617Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"client":"1","":"downstairs","session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1682025-03-07T18:19:05.274Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.195618541Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"client":"1","":"downstairs","session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1692025-03-07T18:19:05.274Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.195659765Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"client":"2","":"downstairs","session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1702025-03-07T18:19:05.274Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.19573355Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"client":"2","":"downstairs","session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1712025-03-07T18:19:05.274Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.195761389Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"io task","client":"2","":"downstairs","session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1722025-03-07T18:19:05.274Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.195878729Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"io task","client":"0","":"downstairs","session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1732025-03-07T18:19:05.274Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.195951333Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"io task","client":"1","":"downstairs","session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1742025-03-07T18:19:05.274Z{"msg":"ds_connection connected from Ok(127.0.0.1:57362)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.195988966Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"io task","client":"2","":"downstairs","session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1752025-03-07T18:19:05.274Z{"msg":"ds_connection connected from Ok(127.0.0.1:62255)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.196012474Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"io task","client":"0","":"downstairs","session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1762025-03-07T18:19:05.275Z{"msg":"ds_connection connected from Ok(127.0.0.1:55472)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.196036102Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"io task","client":"1","":"downstairs","session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1772025-03-07T18:19:05.280Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.19644586Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"client":"1","":"downstairs","session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1782025-03-07T18:19:05.280Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.19650369Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"client":"0","":"downstairs","session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1792025-03-07T18:19:05.280Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.196535941Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"client":"2","":"downstairs","session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1802025-03-07T18:19:05.280Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.197084316Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"client":"1","":"downstairs","session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1812025-03-07T18:19:05.280Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.197173606Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"client":"0","":"downstairs","session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1822025-03-07T18:19:05.280Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.197291316Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"client":"2","":"downstairs","session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1832025-03-07T18:19:05.280Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.19745264Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1842025-03-07T18:19:05.280Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.197592928Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1852025-03-07T18:19:05.280Z{"msg":"[0]R flush_numbers: [3997, 3997, 3997, 3997, 3997]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.19771828Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"downstairs","":"downstairs","session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1862025-03-07T18:19:05.280Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.197755222Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"downstairs","":"downstairs","session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1872025-03-07T18:19:05.280Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.197773388Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"downstairs","":"downstairs","session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1882025-03-07T18:19:05.281Z{"msg":"[1]R flush_numbers: [3997, 3997, 3997, 3997, 3997]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.197796316Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"downstairs","":"downstairs","session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1892025-03-07T18:19:05.281Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.197811341Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"downstairs","":"downstairs","session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1902025-03-07T18:19:05.281Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.197832628Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"downstairs","":"downstairs","session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1912025-03-07T18:19:05.281Z{"msg":"[2]R flush_numbers: [3997, 3997, 3997, 3997, 3997]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.197846683Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"downstairs","":"downstairs","session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1922025-03-07T18:19:05.281Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.197870591Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"downstairs","":"downstairs","session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1932025-03-07T18:19:05.281Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.197888107Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"downstairs","":"downstairs","session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1942025-03-07T18:19:05.281Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.197915927Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"downstairs","":"downstairs","session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1952025-03-07T18:19:05.281Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.197929691Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"downstairs","":"downstairs","session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1962025-03-07T18:19:05.281Z{"msg":"Next flush: 3998","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.197943586Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"downstairs","":"downstairs","session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1972025-03-07T18:19:05.281Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.197970865Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"downstairs","":"downstairs","session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1982025-03-07T18:19:05.281Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.197997124Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
1992025-03-07T18:19:05.281Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.198062046Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"client":"0","":"downstairs","session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
2002025-03-07T18:19:05.281Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.198145164Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"client":"1","":"downstairs","session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
2012025-03-07T18:19:05.281Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.198413395Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"client":"2","":"downstairs","session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
2022025-03-07T18:19:05.281Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.198481067Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
2032025-03-07T18:19:05.281Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.198523912Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
2042025-03-07T18:19:05.281Z{"msg":"8bee2d88-2c7d-45a0-a52a-39646ec755e3 is now active with session: ca73f76b-eb90-4d9f-92cb-56e78ffffca5","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.198561365Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
2052025-03-07T18:19:05.281Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.19857808Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"session_id":"ca73f76b-eb90-4d9f-92cb-56e78ffffca5"}
2062025-03-07T18:19:05.281Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.19860578Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237}
2072025-03-07T18:19:05.281Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.198618054Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237}
2082025-03-07T18:19:05.281Z{"msg":"8bee2d88-2c7d-45a0-a52a-39646ec755e3 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.19863774Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2092025-03-07T18:19:05.281Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.198655106Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"client":"0","":"downstairs","session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2102025-03-07T18:19:05.281Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.198687747Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"client":"0","":"downstairs","session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2112025-03-07T18:19:05.281Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.198708124Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"client":"1","":"downstairs","session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2122025-03-07T18:19:05.281Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.198735933Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"client":"1","":"downstairs","session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2132025-03-07T18:19:05.281Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.198751629Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"client":"2","":"downstairs","session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2142025-03-07T18:19:05.281Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.198765894Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"client":"2","":"downstairs","session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2152025-03-07T18:19:05.282Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.198791052Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"io task","client":"2","":"downstairs","session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2162025-03-07T18:19:05.282Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.198830715Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"io task","client":"0","":"downstairs","session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2172025-03-07T18:19:05.282Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.198879202Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"io task","client":"1","":"downstairs","session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2182025-03-07T18:19:05.282Z{"msg":"ds_connection connected from Ok(127.0.0.1:48629)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.198896067Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"io task","client":"2","":"downstairs","session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2192025-03-07T18:19:05.282Z{"msg":"ds_connection connected from Ok(127.0.0.1:46227)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.198909612Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"io task","client":"1","":"downstairs","session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2202025-03-07T18:19:05.282Z{"msg":"ds_connection connected from Ok(127.0.0.1:44024)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.198923047Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"io task","client":"0","":"downstairs","session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2212025-03-07T18:19:05.282Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.198937481Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"client":"2","":"downstairs","session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2222025-03-07T18:19:05.282Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.198982537Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"client":"1","":"downstairs","session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2232025-03-07T18:19:05.282Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.199000022Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"client":"0","":"downstairs","session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2242025-03-07T18:19:05.282Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.199091063Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"client":"2","":"downstairs","session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2252025-03-07T18:19:05.282Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.199236472Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"client":"1","":"downstairs","session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2262025-03-07T18:19:05.282Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.199441061Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2272025-03-07T18:19:05.283Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.199523589Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"client":"0","":"downstairs","session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2282025-03-07T18:19:05.283Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.199701549Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2292025-03-07T18:19:05.283Z{"msg":"[0]R flush_numbers: [5317, 5310, 5317, 5317, 5317]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.19990993Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"downstairs","":"downstairs","session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2302025-03-07T18:19:05.283Z{"msg":"[0]R generation: [14, 11, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.199937549Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"downstairs","":"downstairs","session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2312025-03-07T18:19:05.283Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.199952344Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"downstairs","":"downstairs","session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2322025-03-07T18:19:05.283Z{"msg":"[1]R flush_numbers: [5317, 5310, 5317, 5317, 5317]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.199965508Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"downstairs","":"downstairs","session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2332025-03-07T18:19:05.283Z{"msg":"[1]R generation: [14, 11, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.199987166Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"downstairs","":"downstairs","session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2342025-03-07T18:19:05.283Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.200002041Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"downstairs","":"downstairs","session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2352025-03-07T18:19:05.283Z{"msg":"[2]R flush_numbers: [5317, 5310, 5317, 5317, 5317]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.200023518Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"downstairs","":"downstairs","session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2362025-03-07T18:19:05.283Z{"msg":"[2]R generation: [14, 11, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.200038213Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"downstairs","":"downstairs","session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2372025-03-07T18:19:05.283Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.200052498Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"downstairs","":"downstairs","session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2382025-03-07T18:19:05.283Z{"msg":"Max found gen is 15","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.200065112Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"downstairs","":"downstairs","session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2392025-03-07T18:19:05.283Z{"msg":"Generation requested: 31 >= found:15","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.200086159Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"downstairs","":"downstairs","session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2402025-03-07T18:19:05.284Z{"msg":"Next flush: 5318","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
2412025-03-07T18:19:05.284Z,"time":"2025-03-07T18:19:04.200100594Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"downstairs","":"downstairs","session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2422025-03-07T18:19:05.284Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.200121581Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"":"downstairs","":"downstairs","session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2432025-03-07T18:19:05.284Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.200134565Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2442025-03-07T18:19:05.284Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.200154572Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"client":"0","":"downstairs","session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2452025-03-07T18:19:05.284Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.200169157Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"client":"1","":"downstairs","session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2462025-03-07T18:19:05.284Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.200189574Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"client":"2","":"downstairs","session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2472025-03-07T18:19:05.284Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.200203509Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2482025-03-07T18:19:05.284Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.200216423Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2492025-03-07T18:19:05.284Z{"msg":"8bee2d88-2c7d-45a0-a52a-39646ec755e3 is now active with session: 36a45bf0-49af-4d67-93a1-55d3287444e2","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.200228827Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2502025-03-07T18:19:05.284Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.200248114Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237,"session_id":"36a45bf0-49af-4d67-93a1-55d3287444e2"}
2512025-03-07T18:19:05.284Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:04.200261768Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237}
2522025-03-07T18:19:05.284ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
2532025-03-07T18:19:05.284ZFill test
2542025-03-07T18:19:05.322ZRead and Verify all blocks (0..100 range:false)
2552025-03-07T18:19:05.400ZWrote out file "/var/tmp/test_up-build/verify_file"
2562025-03-07T18:19:05.400ZCLIENT: Tests done. All submitted work has been ACK'd
2572025-03-07T18:19:05.403Z----------------------------------------------------------------
2582025-03-07T18:19:05.403Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2592025-03-07T18:19:05.403ZStates: Active Active Active
2602025-03-07T18:19:05.403ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2612025-03-07T18:19:05.403Z 1002 Acked Read 50 Done Done Done false
2622025-03-07T18:19:05.403Z STATES DS:0 DS:1 DS:2 TOTAL
2632025-03-07T18:19:05.404Z Sent 0 0 0 0
2642025-03-07T18:19:05.404Z Done 1 1 1 3
2652025-03-07T18:19:05.404Z Skipped 0 0 0 0
2662025-03-07T18:19:05.404Z Error 0 0 0 0
2672025-03-07T18:19:05.404ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2682025-03-07T18:19:05.404ZDownstairs last five completed: 1001 1000
2692025-03-07T18:19:05.404ZUpstairs last five completed: 1002 1001 1000
2702025-03-07T18:19:05.404Z----------------------------------------------------------------
2712025-03-07T18:19:05.404Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2722025-03-07T18:19:05.404ZStates: Active Active Active
2732025-03-07T18:19:05.405ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2742025-03-07T18:19:05.405Z 1002 Acked Read 50 Done Done Done false
2752025-03-07T18:19:05.405Z STATES DS:0 DS:1 DS:2 TOTAL
2762025-03-07T18:19:05.405Z Sent 0 0 0 0
2772025-03-07T18:19:05.405Z Done 1 1 1 3
2782025-03-07T18:19:05.405Z Skipped 0 0 0 0
2792025-03-07T18:19:05.405Z Error 0 0 0 0
2802025-03-07T18:19:05.405ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2812025-03-07T18:19:05.405ZDownstairs last five completed: 1001 1000
2822025-03-07T18:19:05.405ZUpstairs last five completed: 1002 1001 1000
2832025-03-07T18:19:05.405ZCLIENT: Up:0 ds:2 act:6
2842025-03-07T18:19:09.400Z----------------------------------------------------------------
2852025-03-07T18:19:09.400Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2862025-03-07T18:19:09.400ZDownstairs last five completed: 1003 1002 1001 1000
2872025-03-07T18:19:09.400ZUpstairs last five completed: 1003 1002 1001 1000
2882025-03-07T18:19:09.400Z----------------------------------------------------------------
2892025-03-07T18:19:09.400Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2902025-03-07T18:19:09.400ZDownstairs last five completed: 1003 1002 1001 1000
2912025-03-07T18:19:09.401ZUpstairs last five completed: 1003 1002 1001 1000
2922025-03-07T18:19:09.404ZCLIENT: Up:0 ds:0 act:6
2932025-03-07T18:19:09.404ZCLIENT: All crucible jobs finished, exiting program
2942025-03-07T18:19:09.404Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:08.324186939Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1237}
2952025-03-07T18:19:09.412ZRepair setup passed
2962025-03-07T18:19:09.415ZCopy the region for /var/tmp/test_up-build/8830
2972025-03-07T18:19:09.419Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2982025-03-07T18:19:09.423Z/var/tmp/bins/crutest fill -g 32 -q --key JBkvRCjQCWaFPGAUVme4+TRlliNFHE34UEl9Bakpqto= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2992025-03-07T18:19:09.476Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.399341874Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"task":"crutest"}
3002025-03-07T18:19:09.479Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.399942657Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"task":"crutest"}
3012025-03-07T18:19:09.479Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.401546928Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"task":"crutest"}
3022025-03-07T18:19:09.479Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.40172905Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243}
3032025-03-07T18:19:09.479Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.401772414Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243}
3042025-03-07T18:19:09.479Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.401795692Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243}
3052025-03-07T18:19:09.479Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.401811458Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243}
3062025-03-07T18:19:09.479Z{"msg":"Crucible c8b54cec-6d30-4d16-80b5-d7328893c9e2 has session id: aa504058-f238-48b3-a4a7-4882c79a4666","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.402438119Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3072025-03-07T18:19:09.479Z{"msg":"Upstairs opts: Upstairs UUID: c8b54cec-6d30-4d16-80b5-d7328893c9e2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.402477462Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3082025-03-07T18:19:09.479Z{"msg":"Crucible stats registered with UUID: c8b54cec-6d30-4d16-80b5-d7328893c9e2","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.402502971Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3092025-03-07T18:19:09.479Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:08.402540284Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3102025-03-07T18:19:09.483Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.404525744Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"task":"crutest"}
3112025-03-07T18:19:09.483Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.404584214Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243}
3122025-03-07T18:19:09.483Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.404617174Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243}
3132025-03-07T18:19:09.483Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.404643243Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243}
3142025-03-07T18:19:09.483Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.404661669Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243}
3152025-03-07T18:19:09.483Z{"msg":"Crucible c8b54cec-6d30-4d16-80b5-d7328893c9e2 has session id: d0974b7b-6492-4e02-92c9-1221903fba85","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.405001194Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3162025-03-07T18:19:09.483Z{"msg":"Upstairs opts: Upstairs UUID: c8b54cec-6d30-4d16-80b5-d7328893c9e2, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.405091374Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3172025-03-07T18:19:09.483Z{"msg":"Crucible stats registered with UUID: c8b54cec-6d30-4d16-80b5-d7328893c9e2","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.405117203Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3182025-03-07T18:19:09.483Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:08.405156697Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3192025-03-07T18:19:09.483Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.405765832Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"task":"crutest"}
3202025-03-07T18:19:09.483Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.405799293Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243}
3212025-03-07T18:19:09.483Z{"msg":"c8b54cec-6d30-4d16-80b5-d7328893c9e2 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.405867187Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3222025-03-07T18:19:09.483Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.405911121Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"client":"0","":"downstairs","session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3232025-03-07T18:19:09.484Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.405971271Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"client":"0","":"downstairs","session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3242025-03-07T18:19:09.484Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.406014976Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"client":"1","":"downstairs","session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3252025-03-07T18:19:09.484Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.406052979Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"client":"1","":"downstairs","session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3262025-03-07T18:19:09.484Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.40608603Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"client":"2","":"downstairs","session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3272025-03-07T18:19:09.484Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.406111669Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"client":"2","":"downstairs","session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3282025-03-07T18:19:09.484Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.406142829Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"io task","client":"2","":"downstairs","session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3292025-03-07T18:19:09.484Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.406166788Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"io task","client":"0","":"downstairs","session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3302025-03-07T18:19:09.484Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.406198548Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"io task","client":"1","":"downstairs","session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3312025-03-07T18:19:09.484Z{"msg":"ds_connection connected from Ok(127.0.0.1:48474)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.406223067Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"io task","client":"2","":"downstairs","session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3322025-03-07T18:19:09.484Z{"msg":"ds_connection connected from Ok(127.0.0.1:47569)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.406257398Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"io task","client":"1","":"downstairs","session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3332025-03-07T18:19:09.484Z{"msg":"ds_connection connected from Ok(127.0.0.1:37159)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.406425535Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"io task","client":"0","":"downstairs","session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3342025-03-07T18:19:09.491Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.406969318Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"client":"1","":"downstairs","session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3352025-03-07T18:19:09.491Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.407005291Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"client":"0","":"downstairs","session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3362025-03-07T18:19:09.491Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.407022786Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"client":"2","":"downstairs","session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3372025-03-07T18:19:09.491Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.407678778Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"client":"1","":"downstairs","session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3382025-03-07T18:19:09.491Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.407728404Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"client":"2","":"downstairs","session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3392025-03-07T18:19:09.491Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.4077751Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"client":"0","":"downstairs","session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3402025-03-07T18:19:09.491Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.408149146Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3412025-03-07T18:19:09.491Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.408184278Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3422025-03-07T18:19:09.491Z{"msg":"[0]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.408199774Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"downstairs","":"downstairs","session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3432025-03-07T18:19:09.491Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.408232755Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"downstairs","":"downstairs","session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3442025-03-07T18:19:09.491Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.408265386Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"downstairs","":"downstairs","session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3452025-03-07T18:19:09.491Z{"msg":"[1]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.408297307Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"downstairs","":"downstairs","session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3462025-03-07T18:19:09.491Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.408329137Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"downstairs","":"downstairs","session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3472025-03-07T18:19:09.491Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.408362029Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"downstairs","":"downstairs","session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3482025-03-07T18:19:09.491Z{"msg":"[2]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.408380865Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"downstairs","":"downstairs","session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3492025-03-07T18:19:09.491Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.408410885Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"downstairs","":"downstairs","session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3502025-03-07T18:19:09.492Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.408428551Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"downstairs","":"downstairs","session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3512025-03-07T18:19:09.492Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.408465964Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"downstairs","":"downstairs","session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3522025-03-07T18:19:09.492Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.408503376Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"downstairs","":"downstairs","session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3532025-03-07T18:19:09.492Z{"msg":"Next flush: 3999","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.408537298Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"downstairs","":"downstairs","session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3542025-03-07T18:19:09.492Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.408571109Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"downstairs","":"downstairs","session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3552025-03-07T18:19:09.492Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.408784351Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3562025-03-07T18:19:09.492Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.408813771Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"client":"0","":"downstairs","session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3572025-03-07T18:19:09.492Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.408849443Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"client":"1","":"downstairs","session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3582025-03-07T18:19:09.492Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.408884705Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"client":"2","":"downstairs","session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3592025-03-07T18:19:09.492Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.408925949Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3602025-03-07T18:19:09.492Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.408954609Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3612025-03-07T18:19:09.492Z{"msg":"c8b54cec-6d30-4d16-80b5-d7328893c9e2 is now active with session: aa504058-f238-48b3-a4a7-4882c79a4666","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.408982428Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3622025-03-07T18:19:09.492Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.40901904Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"session_id":"aa504058-f238-48b3-a4a7-4882c79a4666"}
3632025-03-07T18:19:09.492Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.409040487Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243}
3642025-03-07T18:19:09.492Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.409066946Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243}
3652025-03-07T18:19:09.492Z{"msg":"c8b54cec-6d30-4d16-80b5-d7328893c9e2 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.409080371Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3662025-03-07T18:19:09.493Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.409093475Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"client":"0","":"downstairs","session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3672025-03-07T18:19:09.493Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.409110671Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"client":"0","":"downstairs","session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3682025-03-07T18:19:09.493Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.40913639Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"client":"1","":"downstairs","session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3692025-03-07T18:19:09.493Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.409161608Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"client":"1","":"downstairs","session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3702025-03-07T18:19:09.493Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.409189678Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"client":"2","":"downstairs","session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3712025-03-07T18:19:09.493Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.409218507Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"client":"2","":"downstairs","session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3722025-03-07T18:19:09.493Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.409277587Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"io task","client":"2","":"downstairs","session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3732025-03-07T18:19:09.493Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.409326944Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"io task","client":"0","":"downstairs","session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3742025-03-07T18:19:09.493Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.409353363Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"io task","client":"1","":"downstairs","session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3752025-03-07T18:19:09.493Z{"msg":"ds_connection connected from Ok(127.0.0.1:40463)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.409375771Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"io task","client":"0","":"downstairs","session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3762025-03-07T18:19:09.493Z{"msg":"ds_connection connected from Ok(127.0.0.1:34409)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.40940258Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"io task","client":"2","":"downstairs","session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3772025-03-07T18:19:09.493Z{"msg":"ds_connection connected from Ok(127.0.0.1:35331)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.409419445Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"io task","client":"1","":"downstairs","session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3782025-03-07T18:19:09.493Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.409454397Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"client":"0","":"downstairs","session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3792025-03-07T18:19:09.493Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.409495121Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"client":"2","":"downstairs","session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3802025-03-07T18:19:09.493Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.409540656Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"client":"1","":"downstairs","session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3812025-03-07T18:19:09.493Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.409591853Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"client":"0","":"downstairs","session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3822025-03-07T18:19:09.493Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.409631477Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"client":"1","":"downstairs","session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3832025-03-07T18:19:09.493Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.409828704Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"client":"2","":"downstairs","session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3842025-03-07T18:19:09.493Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.409984876Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3852025-03-07T18:19:09.493Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.410021709Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3862025-03-07T18:19:09.493Z{"msg":"[0]R flush_numbers: [5318, 5318, 5318, 5318, 5318]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.410159215Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"downstairs","":"downstairs","session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3872025-03-07T18:19:09.493Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.410198658Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"downstairs","":"downstairs","session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3882025-03-07T18:19:09.493Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.410224847Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"downstairs","":"downstairs","session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3892025-03-07T18:19:09.493Z{"msg":"[1]R flush_numbers: [5318, 5318, 5318, 5318, 5318]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.410255717Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"downstairs","":"downstairs","session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3902025-03-07T18:19:09.493Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.410280476Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"downstairs","":"downstairs","session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3912025-03-07T18:19:09.493Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.410304154Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"downstairs","":"downstairs","session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3922025-03-07T18:19:09.493Z{"msg":"[2]R flush_numbers: [5318, 5318, 5318, 5318, 5318]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.410333614Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"downstairs","":"downstairs","session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3932025-03-07T18:19:09.493Z{"msg"Wait for a query_work_queue command to finish before sending IO
3942025-03-07T18:19:09.493Z:"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.410372237Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"downstairs","":"downstairs","session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3952025-03-07T18:19:09.493Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.410397105Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"downstairs","":"downstairs","session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3962025-03-07T18:19:09.493Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.410424614Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"downstairs","":"downstairs","session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3972025-03-07T18:19:09.493Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.410447973Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"downstairs","":"downstairs","session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3982025-03-07T18:19:09.493Z{"msg":"Next flush: 5319","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.410474831Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"downstairs","":"downstairs","session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
3992025-03-07T18:19:09.493Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.410497959Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"":"downstairs","":"downstairs","session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
4002025-03-07T18:19:09.493Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.410524698Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
4012025-03-07T18:19:09.493Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.410547026Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"client":"0","":"downstairs","session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
4022025-03-07T18:19:09.493Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.410575075Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"client":"1","":"downstairs","session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
4032025-03-07T18:19:09.493Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.410598003Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"client":"2","":"downstairs","session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
4042025-03-07T18:19:09.493Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.410624492Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
4052025-03-07T18:19:09.493Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.410646019Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
4062025-03-07T18:19:09.494Z{"msg":"c8b54cec-6d30-4d16-80b5-d7328893c9e2 is now active with session: d0974b7b-6492-4e02-92c9-1221903fba85","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.410672788Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
4072025-03-07T18:19:09.494Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.410694196Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243,"session_id":"d0974b7b-6492-4e02-92c9-1221903fba85"}
4082025-03-07T18:19:09.494Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:08.410724276Z","hostname":"Disk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
4092025-03-07T18:19:09.494Zw-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1243}
4102025-03-07T18:19:09.494ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4112025-03-07T18:19:09.494ZFill test
4122025-03-07T18:19:09.529ZRead and Verify all blocks (0..100 range:false)
4132025-03-07T18:19:09.609ZWrote out file "/var/tmp/test_up-build/verify_file"
4142025-03-07T18:19:09.609ZCLIENT: Tests done. All submitted work has been ACK'd
4152025-03-07T18:19:09.612Z----------------------------------------------------------------
4162025-03-07T18:19:09.612Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4172025-03-07T18:19:09.612ZStates: Active Active Active
4182025-03-07T18:19:09.612ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4192025-03-07T18:19:09.612Z 1002 Acked Read 50 Done Done Done false
4202025-03-07T18:19:09.612Z STATES DS:0 DS:1 DS:2 TOTAL
4212025-03-07T18:19:09.612Z Sent 0 0 0 0
4222025-03-07T18:19:09.612Z Done 1 1 1 3
4232025-03-07T18:19:09.612Z Skipped 0 0 0 0
4242025-03-07T18:19:09.612Z Error 0 0 0 0
4252025-03-07T18:19:09.612ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4262025-03-07T18:19:09.612ZDownstairs last five completed: 1001 1000
4272025-03-07T18:19:09.612ZUpstairs last five completed: 1002 1001 1000
4282025-03-07T18:19:09.612Z----------------------------------------------------------------
4292025-03-07T18:19:09.612Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4302025-03-07T18:19:09.612ZStates: Active Active Active
4312025-03-07T18:19:09.612ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4322025-03-07T18:19:09.612Z 1002 Acked Read 50 Done Done Done false
4332025-03-07T18:19:09.612Z STATES DS:0 DS:1 DS:2 TOTAL
4342025-03-07T18:19:09.612Z Sent 0 0 0 0
4352025-03-07T18:19:09.612Z Done 1 1 1 3
4362025-03-07T18:19:09.612Z Skipped 0 0 0 0
4372025-03-07T18:19:09.613Z Error 0 0 0 0
4382025-03-07T18:19:09.613ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4392025-03-07T18:19:09.613ZDownstairs last five completed: 1001 1000
4402025-03-07T18:19:09.613ZUpstairs last five completed: 1002 1001 1000
4412025-03-07T18:19:09.613ZCLIENT: Up:0 ds:2 act:6
4422025-03-07T18:19:13.609Z----------------------------------------------------------------
4432025-03-07T18:19:13.609Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4442025-03-07T18:19:13.609ZDownstairs last five completed: 1003 1002 1001 1000
4452025-03-07T18:19:13.609ZUpstairs last five completed: 1003 1002 1001 1000
4462025-03-07T18:19:13.613Z----------------------------------------------------------------
4472025-03-07T18:19:13.613Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4482025-03-07T18:19:13.613ZDownstairs last five completed: 1003 1002 1001 1000
4492025-03-07T18:19:13.613ZUpstairs last five completed: 1003 1002 1001 1000
4502025-03-07T18:19:13.613ZCLIENT: Up:0 ds:0 act:6
4512025-03-07T18:19:13.613ZCLIENT: All crucible jobs finished, exiting program
4522025-03-07T18:19:13.617ZRepair part 1 passed
4532025-03-07T18:19:13.620Z
4542025-03-07T18:19:13.620ZKill the current downstairs
4552025-03-07T18:19:13.781ZDownstairs 2 stopped
4562025-03-07T18:19:13.781Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
4572025-03-07T18:19:13.784ZNow put back the original so we have a mismatch
4582025-03-07T18:19:13.784Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
4592025-03-07T18:19:13.788ZRestart downstairs with old directory
4602025-03-07T18:19:13.838Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4612025-03-07T18:19:13.855ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4622025-03-07T18:19:13.858Z 0 000-009 32 32 31 3999 3999 3998 F F F <---
4632025-03-07T18:19:13.858Z 1 010-019 32 32 31 3999 3999 3998 F F F <---
4642025-03-07T18:19:13.858Z 2 020-029 32 32 31 3999 3999 3998 F F F <---
4652025-03-07T18:19:13.858Z 3 030-039 32 32 31 3999 3999 3998 F F F <---
4662025-03-07T18:19:13.858Z 4 040-049 32 32 31 3999 3999 3998 F F F <---
4672025-03-07T18:19:13.858ZMax gen: 32, Max flush: 3999
4682025-03-07T18:19:13.858ZError: Difference in extent metadata found!
4692025-03-07T18:19:13.861Zdump test found error as expected
4702025-03-07T18:19:13.861Z
4712025-03-07T18:19:13.861Z
4722025-03-07T18:19:13.861Z/var/tmp/bins/crutest verify --range -g 33 -q --key JBkvRCjQCWaFPGAUVme4+TRlliNFHE34UEl9Bakpqto= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
4732025-03-07T18:19:13.915Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.839795984Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"task":"crutest"}
4742025-03-07T18:19:13.918Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.840397727Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"task":"crutest"}
4752025-03-07T18:19:13.918Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.842114937Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"task":"crutest"}
4762025-03-07T18:19:13.918Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.842300459Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251}
4772025-03-07T18:19:13.918Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.842344884Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251}
4782025-03-07T18:19:13.918Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.842368542Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251}
4792025-03-07T18:19:13.918Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.842384937Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251}
4802025-03-07T18:19:13.918Z{"msg":"Crucible ca09b4b8-b2db-4705-aace-2a4bae218d6b has session id: 1edd7b9b-ac10-4ad7-8f42-64a878f808d6","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.84292706Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
4812025-03-07T18:19:13.918Z{"msg":"Upstairs opts: Upstairs UUID: ca09b4b8-b2db-4705-aace-2a4bae218d6b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.842982879Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
4822025-03-07T18:19:13.918Z{"msg":"Crucible stats registered with UUID: ca09b4b8-b2db-4705-aace-2a4bae218d6b","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.843009988Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
4832025-03-07T18:19:13.918Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:12.843032306Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
4842025-03-07T18:19:13.922Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.845132575Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"task":"crutest"}
4852025-03-07T18:19:13.922Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.845222215Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251}
4862025-03-07T18:19:13.922Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.845261628Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251}
4872025-03-07T18:19:13.922Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.845275673Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251}
4882025-03-07T18:19:13.922Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.845297531Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251}
4892025-03-07T18:19:13.922Z{"msg":"Crucible ca09b4b8-b2db-4705-aace-2a4bae218d6b has session id: 03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.845674788Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
4902025-03-07T18:19:13.922Z{"msg":"Upstairs opts: Upstairs UUID: ca09b4b8-b2db-4705-aace-2a4bae218d6b, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.845715281Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
4912025-03-07T18:19:13.922Z{"msg":"Crucible stats registered with UUID: ca09b4b8-b2db-4705-aace-2a4bae218d6b","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.84574084Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
4922025-03-07T18:19:13.922Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:12.845763198Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
4932025-03-07T18:19:13.922Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.846388839Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"task":"crutest"}
4942025-03-07T18:19:13.922Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.846407605Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251}
4952025-03-07T18:19:13.922Z{"msg":"ca09b4b8-b2db-4705-aace-2a4bae218d6b active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.846517272Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
4962025-03-07T18:19:13.922Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.846555915Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"0","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
4972025-03-07T18:19:13.922Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.846613465Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"0","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
4982025-03-07T18:19:13.922Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.846665592Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"1","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
4992025-03-07T18:19:13.922Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.846704775Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"1","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5002025-03-07T18:19:13.922Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.846729734Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"2","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5012025-03-07T18:19:13.922Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.846760064Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"2","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5022025-03-07T18:19:13.922Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.846783962Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"io task","client":"2","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5032025-03-07T18:19:13.923Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.84680683Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"io task","client":"0","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5042025-03-07T18:19:13.923Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.846829437Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"io task","client":"1","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5052025-03-07T18:19:13.923Z{"msg":"ds_connection connected from Ok(127.0.0.1:33079)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.846851355Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"io task","client":"0","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5062025-03-07T18:19:13.923Z{"msg":"ds_connection connected from Ok(127.0.0.1:58661)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.846872732Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"io task","client":"1","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5072025-03-07T18:19:13.923Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:12.846913896Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"io task","client":"2","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5082025-03-07T18:19:13.923Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:12.846942836Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"io task","client":"2","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5092025-03-07T18:19:13.923Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:12.847045791Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5102025-03-07T18:19:13.923Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.847084294Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"io task","client":"2","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5112025-03-07T18:19:13.927Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.847593245Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"0","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5122025-03-07T18:19:13.927Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.847630758Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"1","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5132025-03-07T18:19:13.927Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.848253368Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"1","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5142025-03-07T18:19:13.927Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.848371048Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"0","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5152025-03-07T18:19:13.927Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.848632946Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5162025-03-07T18:19:13.927Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:12.848771413Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5172025-03-07T18:19:23.920Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.847809397Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"io task","client":"2","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5182025-03-07T18:19:23.920Z{"msg":"ds_connection connected from Ok(127.0.0.1:60695)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.847892585Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"io task","client":"2","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5192025-03-07T18:19:23.926Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.849037732Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"2","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5202025-03-07T18:19:23.926Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.849598491Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"2","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5212025-03-07T18:19:23.926Z{"msg":"[0]R flush_numbers: [3999, 3999, 3999, 3999, 3999]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.849996585Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"downstairs","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5222025-03-07T18:19:23.926Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.850042741Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"downstairs","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5232025-03-07T18:19:23.926Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.850080044Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"downstairs","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5242025-03-07T18:19:23.926Z{"msg":"[1]R flush_numbers: [3999, 3999, 3999, 3999, 3999]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.850119787Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"downstairs","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5252025-03-07T18:19:23.926Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.85016018Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"downstairs","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5262025-03-07T18:19:23.926Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.850199544Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"downstairs","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5272025-03-07T18:19:23.926Z{"msg":"[2]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.850236646Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"downstairs","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5282025-03-07T18:19:23.926Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.850254122Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"downstairs","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5292025-03-07T18:19:23.926Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.850279861Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"downstairs","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5302025-03-07T18:19:23.926Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.850296246Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"downstairs","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5312025-03-07T18:19:23.926Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.850320735Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"downstairs","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5322025-03-07T18:19:23.926Z{"msg":"Next flush: 4000","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.85033711Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"downstairs","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5332025-03-07T18:19:23.926Z{"msg":"extent 0 needs reconciliation: ExtentMetadata { gen: 32, flush: 3999, dirty: false } ExtentMetadata { gen: 32, flush: 3999, dirty: false } ExtentMetadata { gen: 31, flush: 3998, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.85036428Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"mend","":"downstairs","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5342025-03-07T18:19:23.926Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 3999, dirty: false }, ExtentMetadata { gen: 32, flush: 3999, dirty: false }, ExtentMetadata { gen: 31, flush: 3998, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.850390538Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"mend","":"downstairs","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5352025-03-07T18:19:23.926Z{"msg":"found dest for source 0 for extent at index 0 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.850416727Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"mend","":"downstairs","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5362025-03-07T18:19:23.926Z{"msg":"extent 1 needs reconciliation: ExtentMetadata { gen: 32, flush: 3999, dirty: false } ExtentMetadata { gen: 32, flush: 3999, dirty: false } ExtentMetadata { gen: 31, flush: 3998, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.850441165Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"mend","":"downstairs","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5372025-03-07T18:19:23.926Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 3999, dirty: false }, ExtentMetadata { gen: 32, flush: 3999, dirty: false }, ExtentMetadata { gen: 31, flush: 3998, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.850457081Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"mend","":"downstairs","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5382025-03-07T18:19:23.926Z{"msg":"found dest for source 0 for extent at index 1 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.850477708Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"mend","":"downstairs","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5392025-03-07T18:19:23.926Z{"msg":"extent 2 needs reconciliation: ExtentMetadata { gen: 32, flush: 3999, dirty: false } ExtentMetadata { gen: 32, flush: 3999, dirty: false } ExtentMetadata { gen: 31, flush: 3998, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.850494213Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"mend","":"downstairs","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5402025-03-07T18:19:23.927Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 3999, dirty: false }, ExtentMetadata { gen: 32, flush: 3999, dirty: false }, ExtentMetadata { gen: 31, flush: 3998, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.850515941Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"mend","":"downstairs","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5412025-03-07T18:19:23.927Z{"msg":"found dest for source 0 for extent at index 2 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.850531156Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"mend","":"downstairs","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5422025-03-07T18:19:23.927Z{"msg":"extent 3 needs reconciliation: ExtentMetadata { gen: 32, flush: 3999, dirty: false } ExtentMetadata { gen: 32, flush: 3999, dirty: false } ExtentMetadata { gen: 31, flush: 3998, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.850553723Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"mend","":"downstairs","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5432025-03-07T18:19:23.927Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 3999, dirty: false }, ExtentMetadata { gen: 32, flush: 3999, dirty: false }, ExtentMetadata { gen: 31, flush: 3998, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.850569069Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"mend","":"downstairs","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5442025-03-07T18:19:23.927Z{"msg":"found dest for source 0 for extent at index 3 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.850586274Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"mend","":"downstairs","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5452025-03-07T18:19:23.927Z{"msg":"extent 4 needs reconciliation: ExtentMetadata { gen: 32, flush: 3999, dirty: false } ExtentMetadata { gen: 32, flush: 3999, dirty: false } ExtentMetadata { gen: 31, flush: 3998, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.850608112Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"mend","":"downstairs","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5462025-03-07T18:19:23.927Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 3999, dirty: false }, ExtentMetadata { gen: 32, flush: 3999, dirty: false }, ExtentMetadata { gen: 31, flush: 3998, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.850643003Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"mend","":"downstairs","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5472025-03-07T18:19:23.927Z{"msg":"found dest for source 0 for extent at index 4 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.850671523Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"mend","":"downstairs","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5482025-03-07T18:19:23.927Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.850708576Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"0","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5492025-03-07T18:19:23.927Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.850755041Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"1","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5502025-03-07T18:19:23.927Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.850785512Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"2","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5512025-03-07T18:19:23.927Z{"msg":"Full repair list: {ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.850823084Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"downstairs","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5522025-03-07T18:19:23.927Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 4000, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 4000, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 4000, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 4000, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 4000, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.850906783Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"downstairs","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5532025-03-07T18:19:23.927Z{"msg":"starting reconciliation 8ea78ae0-3719-4722-8a43-6b0ff3e1294f: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.850925079Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"downstairs","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5542025-03-07T18:19:23.927Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.850938643Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"2","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5552025-03-07T18:19:23.932Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.86017165Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"2","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5562025-03-07T18:19:23.936Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.863871718Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"2","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5572025-03-07T18:19:23.939Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.867467012Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"2","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5582025-03-07T18:19:23.943Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.871089044Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"2","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5592025-03-07T18:19:23.946Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.874207957Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"downstairs","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5602025-03-07T18:19:23.949Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.874252592Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"0","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5612025-03-07T18:19:23.950Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.874293276Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"1","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5622025-03-07T18:19:23.950Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.874314843Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"2","":"downstairs","session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5632025-03-07T18:19:23.950Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.874351055Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5642025-03-07T18:19:23.950Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.874382796Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5652025-03-07T18:19:23.950Z{"msg":"ca09b4b8-b2db-4705-aace-2a4bae218d6b is now active with session: 1edd7b9b-ac10-4ad7-8f42-64a878f808d6","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.874399101Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"session_id":"1edd7b9b-ac10-4ad7-8f42-64a878f808d6"}
5662025-03-07T18:19:23.950Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.874419428Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251}
5672025-03-07T18:19:23.950Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.874431932Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251}
5682025-03-07T18:19:23.950Z{"msg":"ca09b4b8-b2db-4705-aace-2a4bae218d6b active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.874445237Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
5692025-03-07T18:19:23.950Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.874472346Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"0","":"downstairs","session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
5702025-03-07T18:19:23.950Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.874500555Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"0","":"downstairs","session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
5712025-03-07T18:19:23.950Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.874533947Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"1","":"downstairs","session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
5722025-03-07T18:19:23.950Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.874559195Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"1","":"downstairs","session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
5732025-03-07T18:19:23.950Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.874585754Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"2","":"downstairs","session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
5742025-03-07T18:19:23.950Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.874613434Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"2","":"downstairs","session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
5752025-03-07T18:19:23.950Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.874640503Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"io task","client":"2","":"downstairs","session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
5762025-03-07T18:19:23.950Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.874668702Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"io task","client":"0","":"downstairs","session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
5772025-03-07T18:19:23.950Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.874695321Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"io task","client":"1","":"downstairs","session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
5782025-03-07T18:19:23.950Z{"msg":"ds_connection connected from Ok(127.0.0.1:54698)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.87471972Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"io task","client":"0","":"downstairs","session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
5792025-03-07T18:19:23.950Z{"msg":"ds_connection connected from Ok(127.0.0.1:33801)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.874744548Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"io task","client":"2","":"downstairs","session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
5802025-03-07T18:19:23.950Z{"msg":"ds_connection connected from Ok(127.0.0.1:35255)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.874768756Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"io task","client":"1","":"downstairs","session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
5812025-03-07T18:19:23.950Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.874934452Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"1","":"downstairs","session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
5822025-03-07T18:19:23.950Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.874975906Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"0","":"downstairs","session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
5832025-03-07T18:19:23.950Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.874992101Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"2","":"downstairs","session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
5842025-03-07T18:19:23.951Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.875469953Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"1","":"downstairs","session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
5852025-03-07T18:19:23.951Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.875568006Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"0","":"downstairs","session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
5862025-03-07T18:19:23.951Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.875721167Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"2","":"downstairs","session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
5872025-03-07T18:19:23.951Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.875879601Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
5882025-03-07T18:19:23.951Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.876015837Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
5892025-03-07T18:19:23.951Z{"msg":"[0]R flush_numbers: [5319, 5319, 5319, 5319, 5319]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.876121763Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"downstairs","":"downstairs","session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
5902025-03-07T18:19:23.951Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.876148982Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"downstairs","":"downstairs","session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
5912025-03-07T18:19:23.951Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.876164777Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"downstairs","":"downstairs","session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
5922025-03-07T18:19:23.952Z{"msg":"[1]R flush_numbers: [5319, 5319, 5319, 5319, 5319]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.876185354Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"downstairs","":"downstairs","session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
5932025-03-07T18:19:23.952Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.876198609Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"downstairs","":"downstairs","session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
5942025-03-07T18:19:23.952Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.876211623Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"downstairs","":"downstairs","session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
5952025-03-07T18:19:23.952Z{"msg":"[2]R flush_numbers: [5319, 5319, 5319, 5319, 5319]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.87623246Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"downstairs","":"downstairs","session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
5962025-03-07T18:19:23.952Z{"msg":"[2]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.876245664Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"downstairs","":"downstairs","session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
5972025-03-07T18:19:23.952Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.876268572Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"downstairs","":"downstairs","session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
5982025-03-07T18:19:23.952Z{"msg":"Max found gen is 33",Wait for a query_work_queue command to finish before sending IO
5992025-03-07T18:19:23.952Z"v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.87629216Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"downstairs","":"downstairs","session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
6002025-03-07T18:19:23.952Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.876314278Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"downstairs","":"downstairs","session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
6012025-03-07T18:19:23.952Z{"msg":"Next flush: 5320","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.876327822Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"downstairs","":"downstairs","session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
6022025-03-07T18:19:23.952Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.876340376Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"":"downstairs","":"downstairs","session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
6032025-03-07T18:19:23.952Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.876360283Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
6042025-03-07T18:19:23.952Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.876372157Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"0","":"downstairs","session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
6052025-03-07T18:19:23.953Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.876384811Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"1","":"downstairs","session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
6062025-03-07T18:19:23.953Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.876397316Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"client":"2","":"downstairs","session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
6072025-03-07T18:19:23.953Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.876417153Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
6082025-03-07T18:19:23.953Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.876429466Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
6092025-03-07T18:19:23.953Z{"msg":"ca09b4b8-b2db-4705-aace-2a4bae218d6b is now active with session: 03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.876441971Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
6102025-03-07T18:19:23.953Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.876454145Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251,"session_id":"03d5a31f-fce1-4d58-82ac-45ce4e5e1a2c"}
6112025-03-07T18:19:23.953Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:22.876465759Z","hostname":"w-01JNRWVM07N8XV2TSYHKMREPCJ","pid":1251}
6122025-03-07T18:19:23.953ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
6132025-03-07T18:19:23.953ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
6142025-03-07T18:19:23.953ZRead and Verify all blocks (0..100 range:true)
6152025-03-07T18:19:24.028ZWrote out file "/var/tmp/test_up-build/verify_file"
6162025-03-07T18:19:24.028ZVerify test completed
6172025-03-07T18:19:24.031ZWrote out file "/var/tmp/test_up-build/verify_file"
6182025-03-07T18:19:24.031ZCLIENT: Tests done. All submitted work has been ACK'd
6192025-03-07T18:19:24.031Z----------------------------------------------------------------
6202025-03-07T18:19:24.031Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6212025-03-07T18:19:24.031ZStates: Active Active Active
6222025-03-07T18:19:24.031ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6232025-03-07T18:19:24.031Z 1000 Acked Read 50 Done Done Done false
6242025-03-07T18:19:24.031Z STATES DS:0 DS:1 DS:2 TOTAL
6252025-03-07T18:19:24.031Z Sent 0 0 0 0
6262025-03-07T18:19:24.031Z Done 1 1 1 3
6272025-03-07T18:19:24.031Z Skipped 0 0 0 0
6282025-03-07T18:19:24.031Z Error 0 0 0 0
6292025-03-07T18:19:24.031ZLast Flush: None None None
6302025-03-07T18:19:24.031ZDownstairs last five completed:
6312025-03-07T18:19:24.031ZUpstairs last five completed: 1000
6322025-03-07T18:19:24.031Z----------------------------------------------------------------
6332025-03-07T18:19:24.031Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6342025-03-07T18:19:24.031ZStates: Active Active Active
6352025-03-07T18:19:24.031ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6362025-03-07T18:19:24.032Z 1000 Acked Read 50 Done Done Done false
6372025-03-07T18:19:24.032Z STATES DS:0 DS:1 DS:2 TOTAL
6382025-03-07T18:19:24.032Z Sent 0 0 0 0
6392025-03-07T18:19:24.032Z Done 1 1 1 3
6402025-03-07T18:19:24.032Z Skipped 0 0 0 0
6412025-03-07T18:19:24.032Z Error 0 0 0 0
6422025-03-07T18:19:24.032ZLast Flush: None None None
6432025-03-07T18:19:24.032ZDownstairs last five completed:
6442025-03-07T18:19:24.032ZUpstairs last five completed: 1000
6452025-03-07T18:19:24.032ZCLIENT: Up:0 ds:2 act:6
6462025-03-07T18:19:28.028Z----------------------------------------------------------------
6472025-03-07T18:19:28.028Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6482025-03-07T18:19:28.028ZDownstairs last five completed: 1001 1000
6492025-03-07T18:19:28.028ZUpstairs last five completed: 1001 1000
6502025-03-07T18:19:28.032Z----------------------------------------------------------------
6512025-03-07T18:19:28.032Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6522025-03-07T18:19:28.032ZDownstairs last five completed: 1001 1000
6532025-03-07T18:19:28.032ZUpstairs last five completed: 1001 1000
6542025-03-07T18:19:28.032ZCLIENT: Up:0 ds:0 act:6
6552025-03-07T18:19:28.032ZCLIENT: All crucible jobs finished, exiting program
6562025-03-07T18:19:28.035ZRepair part 2 passed
6572025-03-07T18:19:28.036Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
6582025-03-07T18:19:28.053ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6592025-03-07T18:19:28.053Z 0 000-009 32 32 32 3999 3999 3999 F F F
6602025-03-07T18:19:28.053Z 1 010-019 32 32 32 3999 3999 3999 F F F
6612025-03-07T18:19:28.056Z 2 020-029 32 32 32 3999 3999 3999 F F F
6622025-03-07T18:19:28.056Z 3 030-039 32 32 32 3999 3999 3999 F F F
6632025-03-07T18:19:28.056Z 4 040-049 32 32 32 3999 3999 3999 F F F
6642025-03-07T18:19:28.056ZMax gen: 32, Max flush: 3999
6652025-03-07T18:19:28.059Zdump test passed
6662025-03-07T18:19:28.059Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
6672025-03-07T18:19:28.072Z Extent 1
6682025-03-07T18:19:28.077ZGEN 32 32 32
6692025-03-07T18:19:28.077ZFLUSH_ID 3999 3999 3999
6702025-03-07T18:19:28.077ZDIRTY
6712025-03-07T18:19:28.077Z
6722025-03-07T18:19:28.077ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
6732025-03-07T18:19:28.077Z010 A A A A A A
6742025-03-07T18:19:28.077Z011 A A A A A A
6752025-03-07T18:19:28.081Z012 A A A A A A
6762025-03-07T18:19:28.082Z013 A A A A A A
6772025-03-07T18:19:28.082Z014 A A A A A A
6782025-03-07T18:19:28.082Z015 A A A A A A
6792025-03-07T18:19:28.085Z016 A A A A A A
6802025-03-07T18:19:28.085Z017 A A A A A A
6812025-03-07T18:19:28.085Z018 A A A A A A
6822025-03-07T18:19:28.089Z019 A A A A A A
6832025-03-07T18:19:28.092Zdump extent test passed
6842025-03-07T18:19:28.092Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
6852025-03-07T18:19:28.105ZExtent 2 Block in extent 0 Actual block 20
6862025-03-07T18:19:28.108Z
6872025-03-07T18:19:28.108Z DATA SHA256 VER
6882025-03-07T18:19:28.108Z------ ---------------------------------------------------------------- ---
6892025-03-07T18:19:28.108Z 0 30c8663e70d4210d68cfb40779db4a338ab39b898d4338e7d8ab18a7dfbf2e6e A
6902025-03-07T18:19:28.108Z 1 30c8663e70d4210d68cfb40779db4a338ab39b898d4338e7d8ab18a7dfbf2e6e A
6912025-03-07T18:19:28.108Z 2 30c8663e70d4210d68cfb40779db4a338ab39b898d4338e7d8ab18a7dfbf2e6e A
6922025-03-07T18:19:28.108Z
6932025-03-07T18:19:28.108ZNONCES 0 1 2 DIFF
6942025-03-07T18:19:28.108Z------ ------------------------ ------------------------ ------------------------ -----
6952025-03-07T18:19:28.108Z 0 abf58d55443694a038023b6c abf58d55443694a038023b6c abf58d55443694a038023b6c
6962025-03-07T18:19:28.108Z
6972025-03-07T18:19:28.108Z TAGS 0 1 2 DIFF
6982025-03-07T18:19:28.108Z------ -------------------------------- -------------------------------- -------------------------------- -----
6992025-03-07T18:19:28.108Z 0 c52ad87a6fde2bc98cb6d752e500c498 c52ad87a6fde2bc98cb6d752e500c498 c52ad87a6fde2bc98cb6d752e500c498
7002025-03-07T18:19:28.108Z
7012025-03-07T18:19:28.108ZHASHES 0 1 2 DIFF
7022025-03-07T18:19:28.108Z------ ---------------- ---------------- ---------------- -----
7032025-03-07T18:19:28.108Z
7042025-03-07T18:19:28.112Zdump block test passed
7052025-03-07T18:19:28.112ZInitial upstairs tests have completed, stopping all downstairs
7062025-03-07T18:19:33.176ZCreating 7 larger downstairs regions
7072025-03-07T18:19:33.362ZStarting 7 downstairs
7082025-03-07T18:19:38.367Zdsc restarted at PID: 1267
7092025-03-07T18:19:38.755ZNow do the replace-reconcile test
7102025-03-07T18:19:38.809ZUsing 8870 for the replacement port
7112025-03-07T18:22:50.188ZCompleted test: replace-reconcile
7122025-03-07T18:22:50.192ZNow do the replace-before-active test
7132025-03-07T18:26:23.796ZCompleted test: replace-before-active
7142025-03-07T18:26:23.799ZAll tests have completed, stopping all downstairs
7152025-03-07T18:26:23.854Z
7162025-03-07T18:26:23.854ZAll Tests have passed
7172025-03-07T18:26:23.857Z13:55 Test duration
7182025-03-07T18:26:23.858Z
7192025-03-07T18:26:23.858Zreal 13:55.061292795
7202025-03-07T18:26:23.858Zuser 18:40.408381576
7212025-03-07T18:26:23.858Zsys 24.799284451
7222025-03-07T18:26:23.858Ztrap 0.530489209
7232025-03-07T18:26:23.858Ztflt 0.043763373
7242025-03-07T18:26:23.858Zdflt 0.051983021
7252025-03-07T18:26:23.858Zkflt 0.001218710
7262025-03-07T18:26:23.858Zlock 25:40:47.486337420
7272025-03-07T18:26:23.858Zslp 1:15:14.137599794
7282025-03-07T18:26:23.858Zlat 2:07.958122099
7292025-03-07T18:26:23.858Zstop 0.201414423
7302025-03-07T18:26:23.858Z+ echo 'test-up-2region-encrypted ends'
7312025-03-07T18:26:23.858Ztest-up-2region-encrypted ends
7322025-03-07T18:26:28.859Zprocess exited: duration 843540 ms, exit code 0
7332025-03-07T18:26:28.859Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
7342025-03-07T18:27:28.880Zexec warning: : stdout descriptor may be held open by a background process; giving up!
7352025-03-07T18:27:28.880Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
7362025-03-07T18:27:28.893Zfound 17 output files
7372025-03-07T18:27:28.893Zuploading: /tmp/test_up-build/dsc-out.txt (53753 bytes)
7382025-03-07T18:27:29.909Zuploaded: /tmp/test_up-build/dsc-out.txt
7392025-03-07T18:27:29.919Zuploading: /tmp/test_up-build/test_up_out.txt (8730316 bytes)
7402025-03-07T18:27:31.030Zuploaded: /tmp/test_up-build/test_up_out.txt
7412025-03-07T18:27:31.030Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (147244 bytes)
7422025-03-07T18:27:32.046Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
7432025-03-07T18:27:32.046Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (144642 bytes)
7442025-03-07T18:27:33.063Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
7452025-03-07T18:27:33.067Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (250985 bytes)
7462025-03-07T18:27:34.081Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
7472025-03-07T18:27:34.085Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (139417 bytes)
7482025-03-07T18:27:35.098Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
7492025-03-07T18:27:35.098Zuploading: /tmp/test_up-build/dsc/downstairs-8850.txt (2497 bytes)
7502025-03-07T18:27:36.109Zuploaded: /tmp/test_up-build/dsc/downstairs-8850.txt
7512025-03-07T18:27:36.109Zuploading: /tmp/test_up-build/dsc/downstairs-8860.txt (36401 bytes)
7522025-03-07T18:27:37.122Zuploaded: /tmp/test_up-build/dsc/downstairs-8860.txt
7532025-03-07T18:27:37.122Zuploading: /tmp/test_up-build/dsc/downstairs-8870.txt (740832 bytes)
7542025-03-07T18:27:38.148Zuploaded: /tmp/test_up-build/dsc/downstairs-8870.txt
7552025-03-07T18:27:38.151Zuploading: /tmp/debug/df.txt (1270 bytes)
7562025-03-07T18:27:39.158Zuploaded: /tmp/debug/df.txt
7572025-03-07T18:27:39.159Zuploading: /tmp/debug/dtrace.txt (958142 bytes)
7582025-03-07T18:27:39.183Zupload warning: file "/tmp/debug/dtrace.txt" changed size mid upload: 958142 -> 1082516
7592025-03-07T18:27:40.190Zuploaded: /tmp/debug/dtrace.txt
7602025-03-07T18:27:40.190Zuploading: /tmp/debug/iostat.txt (268565 bytes)
7612025-03-07T18:27:40.204Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 268565 -> 271843
7622025-03-07T18:27:41.211Zuploaded: /tmp/debug/iostat.txt
7632025-03-07T18:27:41.211Zuploading: /tmp/debug/mpstat.txt (655038 bytes)
7642025-03-07T18:27:41.229Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 655038 -> 663762
7652025-03-07T18:27:42.239Zuploaded: /tmp/debug/mpstat.txt
7662025-03-07T18:27:42.242Zuploading: /tmp/debug/paging.txt (116191 bytes)
7672025-03-07T18:27:42.249Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 116191 -> 117818
7682025-03-07T18:27:43.256Zuploaded: /tmp/debug/paging.txt
7692025-03-07T18:27:43.259Zuploading: /tmp/debug/prstat.txt (1275608 bytes)
7702025-03-07T18:27:43.282Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 1275608 -> 1293662
7712025-03-07T18:27:44.288Zuploaded: /tmp/debug/prstat.txt
7722025-03-07T18:27:44.292Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
7732025-03-07T18:27:45.298Zuploaded: /tmp/debug/psrinfo.txt
7742025-03-07T18:27:45.298Zuploading: /tmp/debug/upstairs-info.txt (185240 bytes)
7752025-03-07T18:27:46.315Zuploaded: /tmp/debug/upstairs-info.txt