01JJMKBH067K2XP9ADBTST91F3: test-up-2region-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01JJMKBZN409MY6GMYG63STQ5D

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12025-01-27T19:34:47.450Zjob dependencies complete; ready to run (waiting for 13 m 31 s)
22025-01-27T19:35:50.760Zjob assigned to worker 01JJMM53SJ2DSW4RAZMQ3Q0NNR [factory aws, i-066d739edd42a8662] (queued for 1 m 3 s)
32025-01-27T19:35:58.984Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42025-01-27T19:36:00.392Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52025-01-27T19:36:00.392Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62025-01-27T19:36:02.164Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72025-01-27T19:36:02.165Zdownloading input: /input/build/work/bins/crutest.gz
82025-01-27T19:36:03.348Zdownloaded input: /input/build/work/bins/crutest.gz
92025-01-27T19:36:03.348Zdownloading input: /input/build/work/bins/dsc.gz
102025-01-27T19:36:04.131Zdownloaded input: /input/build/work/bins/dsc.gz
112025-01-27T19:36:04.134Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122025-01-27T19:36:04.196Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132025-01-27T19:36:04.196Zdownloading input: /input/build/work/scripts/test_ds.sh
142025-01-27T19:36:04.212Zdownloaded input: /input/build/work/scripts/test_ds.sh
152025-01-27T19:36:04.213Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162025-01-27T19:36:04.231Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172025-01-27T19:36:04.231Zdownloading input: /input/build/work/scripts/test_repair.sh
182025-01-27T19:36:04.252Zdownloaded input: /input/build/work/scripts/test_repair.sh
192025-01-27T19:36:04.252Zdownloading input: /input/build/work/scripts/test_replay.sh
202025-01-27T19:36:04.272Zdownloaded input: /input/build/work/scripts/test_replay.sh
212025-01-27T19:36:04.272Zdownloading input: /input/build/work/scripts/test_up.sh
222025-01-27T19:36:04.289Zdownloaded input: /input/build/work/scripts/test_up.sh
232025-01-27T19:36:04.289Zdownloading input: /input/build/work/scripts/upstairs_info.d
242025-01-27T19:36:04.308Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252025-01-27T19:36:04.311Zdownloading input: /input/build/tmp/cargo-test-out.log
262025-01-27T19:36:04.980Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272025-01-27T19:36:04.980Zstarting task 0: "setup"
282025-01-27T19:36:04.984Z++ uname -s
292025-01-27T19:36:04.984Z+ kern=SunOS
302025-01-27T19:36:04.984Z+ case "$kern" in
312025-01-27T19:36:04.984Z+ groupadd -g 12345 build
322025-01-27T19:36:04.988Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332025-01-27T19:36:06.994Z+ zfs create -o mountpoint=/work rpool/work
342025-01-27T19:36:07.559Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352025-01-27T19:36:07.562Z+ home_fs=zfs
362025-01-27T19:36:07.562Z+ [[ zfs == autofs ]]
372025-01-27T19:36:07.562Z+ mkdir -p /home/build
382025-01-27T19:36:07.565Z+ chown build:build /home/build /work
392025-01-27T19:36:08.566Z+ chmod 0700 /home/build /work
402025-01-27T19:36:08.568Zprocess exited: duration 3596 ms, exit code 0
 
412025-01-27T19:36:08.575Zstarting task 1: "authentication"
422025-01-27T19:36:08.591Zprocess exited: duration 17 ms, exit code 0
 
432025-01-27T19:36:08.597Zstarting task 2: "build"
442025-01-27T19:36:08.602Z+ banner cores
452025-01-27T19:36:08.623Z
462025-01-27T19:36:08.623Z #### #### ##### ###### ####
472025-01-27T19:36:08.623Z # # # # # # # #
482025-01-27T19:36:08.623Z # # # # # ##### ####
492025-01-27T19:36:08.623Z # # # ##### # #
502025-01-27T19:36:08.623Z # # # # # # # # #
512025-01-27T19:36:08.623Z #### #### # # ###### ####
522025-01-27T19:36:08.623Z
532025-01-27T19:36:08.627Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542025-01-27T19:36:08.631Z+ banner unpack
552025-01-27T19:36:08.634Z
562025-01-27T19:36:08.634Z # # # # ##### ## #### # #
572025-01-27T19:36:08.634Z # # ## # # # # # # # # #
582025-01-27T19:36:08.634Z # # # # # # # # # # ####
592025-01-27T19:36:08.634Z # # # # # ##### ###### # # #
602025-01-27T19:36:08.634Z # # # ## # # # # # # #
612025-01-27T19:36:08.634Z #### # # # # # #### # #
622025-01-27T19:36:08.634Z
632025-01-27T19:36:08.634Z+ mkdir -p /var/tmp/bins
642025-01-27T19:36:08.637Z+ for t in "$input/bins/"*.gz
652025-01-27T19:36:08.637Z++ basename /input/build/work/bins/crucible-downstairs.gz
662025-01-27T19:36:08.639Z+ b=crucible-downstairs.gz
672025-01-27T19:36:08.639Z+ b=crucible-downstairs
682025-01-27T19:36:08.639Z+ gunzip
692025-01-27T19:36:10.896Z+ chmod +x /var/tmp/bins/crucible-downstairs
702025-01-27T19:36:10.898Z+ for t in "$input/bins/"*.gz
712025-01-27T19:36:10.898Z++ basename /input/build/work/bins/crucible-hammer.gz
722025-01-27T19:36:10.898Z+ b=crucible-hammer.gz
732025-01-27T19:36:10.898Z+ b=crucible-hammer
742025-01-27T19:36:10.898Z+ gunzip
752025-01-27T19:36:13.093Z+ chmod +x /var/tmp/bins/crucible-hammer
762025-01-27T19:36:13.096Z+ for t in "$input/bins/"*.gz
772025-01-27T19:36:13.096Z++ basename /input/build/work/bins/crutest.gz
782025-01-27T19:36:13.099Z+ b=crutest.gz
792025-01-27T19:36:13.099Z+ b=crutest
802025-01-27T19:36:13.099Z+ gunzip
812025-01-27T19:36:15.395Z+ chmod +x /var/tmp/bins/crutest
822025-01-27T19:36:15.397Z+ for t in "$input/bins/"*.gz
832025-01-27T19:36:15.400Z++ basename /input/build/work/bins/dsc.gz
842025-01-27T19:36:15.400Z+ b=dsc.gz
852025-01-27T19:36:15.400Z+ b=dsc
862025-01-27T19:36:15.400Z+ gunzip
872025-01-27T19:36:16.812Z+ chmod +x /var/tmp/bins/dsc
882025-01-27T19:36:16.815Z+ export BINDIR=/var/tmp/bins
892025-01-27T19:36:16.815Z+ BINDIR=/var/tmp/bins
902025-01-27T19:36:16.815Z+ jobpid=1139
912025-01-27T19:36:16.819Z+ echo 'Setup debug logging'
922025-01-27T19:36:16.819ZSetup debug logging
932025-01-27T19:36:16.819Z+ mkdir /tmp/debug
942025-01-27T19:36:16.819Z+ sleep 3600
952025-01-27T19:36:16.819Z+ psrinfo -v
962025-01-27T19:36:16.822Z+ df -h
972025-01-27T19:36:16.846Z+ prstat -d d -mLc 1
982025-01-27T19:36:16.848Z+ iostat -T d -xn 1
992025-01-27T19:36:16.848Z+ mpstat -T d 1
1002025-01-27T19:36:16.848Z+ vmstat -T d -p 1
1012025-01-27T19:36:16.848Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1022025-01-27T19:36:16.851Z+ banner test_up_2r_encrypted
1032025-01-27T19:36:16.851Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1042025-01-27T19:36:16.851Z #####
1052025-01-27T19:36:16.851Z ##### ###### #### ##### # # ##### # # #####
1062025-01-27T19:36:16.851Z # # # # # # # # # # #
1072025-01-27T19:36:16.851Z # ##### #### # # # # # ##### # #
1082025-01-27T19:36:16.851Z # # # # # # ##### # #####
1092025-01-27T19:36:16.851Z # # # # # # # # # # #
1102025-01-27T19:36:16.851Z # ###### #### # ####### #### # ####### ####### # #
1112025-01-27T19:36:16.851Z
1122025-01-27T19:36:16.855Z+ ptime -m bash /input/build/work/scripts/test_up.sh -r 2 -N encrypted
1132025-01-27T19:36:16.862Z/input/build/work
1142025-01-27T19:36:16.892ZUsing 2 region sets
1152025-01-27T19:36:16.892ZTurn off color for downstairs dump
1162025-01-27T19:36:16.925ZUpstairs using key: XIsiXvqqCpigCMpSEKjd+Elr+bwxGR3bvpkVLYo+Vjs=
1172025-01-27T19:36:16.928Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1182025-01-27T19:36:16.928ZCreating 6 downstairs regions
1192025-01-27T19:36:17.313ZStarting 6 downstairs
1202025-01-27T19:36:17.316Zdsc started at PID: 1187
1212025-01-27T19:36:22.672ZDisable automatic restart on all downstairs
1222025-01-27T19:36:22.727Z
1232025-01-27T19:36:22.727ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1242025-01-27T19:36:22.731ZRunning test: span
1252025-01-27T19:36:26.870ZCompleted test: span
1262025-01-27T19:36:31.879ZRunning test: big
1272025-01-27T19:36:36.474ZCompleted test: big
1282025-01-27T19:36:41.482ZRunning test: dep
1292025-01-27T19:37:15.069ZCompleted test: dep
1302025-01-27T19:37:20.077ZRunning test: balloon
1312025-01-27T19:41:33.914ZCompleted test: balloon
1322025-01-27T19:41:38.923ZRunning test: deactivate
1332025-01-27T19:42:33.605ZCompleted test: deactivate
1342025-01-27T19:42:38.612ZRunning hammer
1352025-01-27T19:42:56.201ZRun repair tests
1362025-01-27T19:42:56.205Z/var/tmp/bins/crutest fill -g 31 -q --key XIsiXvqqCpigCMpSEKjd+Elr+bwxGR3bvpkVLYo+Vjs= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1372025-01-27T19:42:56.265Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.684362005Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"task":"crutest"}
1382025-01-27T19:42:56.268Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.685021689Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"task":"crutest"}
1392025-01-27T19:42:56.268Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.686734038Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"task":"crutest"}
1402025-01-27T19:42:56.268Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.68695437Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230}
1412025-01-27T19:42:56.268Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d77aa90b6afe7d7ddb1dc63211fa4216c8866a90\",\n git_commit_timestamp: \"2025-01-27T19:20:16.000000000Z\",\n git_branch: \"mkeeter/shrink-replay-buffer\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.687002591Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230}
1422025-01-27T19:42:56.268Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.687026627Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230}
1432025-01-27T19:42:56.268Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.687041984Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230}
1442025-01-27T19:42:56.268Z{"msg":"Crucible 3bfd5f33-f958-4404-b47b-f5fedf3ce48a has session id: d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.687616703Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1452025-01-27T19:42:56.268Z{"msg":"Upstairs opts: Upstairs UUID: 3bfd5f33-f958-4404-b47b-f5fedf3ce48a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.687670014Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1462025-01-27T19:42:56.268Z{"msg":"Crucible stats registered with UUID: 3bfd5f33-f958-4404-b47b-f5fedf3ce48a","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.687696129Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1472025-01-27T19:42:56.268Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-27T19:42:54.687717426Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1482025-01-27T19:42:56.271Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.689882376Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"task":"crutest"}
1492025-01-27T19:42:56.271Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.689988257Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230}
1502025-01-27T19:42:56.271Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d77aa90b6afe7d7ddb1dc63211fa4216c8866a90\",\n git_commit_timestamp: \"2025-01-27T19:20:16.000000000Z\",\n git_branch: \"mkeeter/shrink-replay-buffer\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.69002998Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230}
1512025-01-27T19:42:56.271Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.690061084Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230}
1522025-01-27T19:42:56.271Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.690082341Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230}
1532025-01-27T19:42:56.271Z{"msg":"Crucible 3bfd5f33-f958-4404-b47b-f5fedf3ce48a has session id: 37d9a885-5354-42c7-8e44-627fd008640a","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.690434189Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
1542025-01-27T19:42:56.272Z{"msg":"Upstairs opts: Upstairs UUID: 3bfd5f33-f958-4404-b47b-f5fedf3ce48a, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.69048315Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
1552025-01-27T19:42:56.272Z{"msg":"Crucible stats registered with UUID: 3bfd5f33-f958-4404-b47b-f5fedf3ce48a","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.690507406Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
1562025-01-27T19:42:56.272Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-27T19:42:54.690537441Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
1572025-01-27T19:42:56.272Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.691327292Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"task":"crutest"}
1582025-01-27T19:42:56.272Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.691368955Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230}
1592025-01-27T19:42:56.272Z{"msg":"3bfd5f33-f958-4404-b47b-f5fedf3ce48a active request set","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.691500401Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1602025-01-27T19:42:56.272Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.691529467Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"client":"0","":"downstairs","session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1612025-01-27T19:42:56.276Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.691555932Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"client":"0","":"downstairs","session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1622025-01-27T19:42:56.276Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.691591705Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"client":"1","":"downstairs","session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1632025-01-27T19:42:56.276Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.691618711Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"client":"1","":"downstairs","session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1642025-01-27T19:42:56.276Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.691654905Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"client":"2","":"downstairs","session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1652025-01-27T19:42:56.276Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.691673801Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"client":"2","":"downstairs","session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1662025-01-27T19:42:56.276Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.691698137Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1672025-01-27T19:42:56.276Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.691722842Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1682025-01-27T19:42:56.276Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.691749658Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1692025-01-27T19:42:56.276Z{"msg":"ds_connection connected from Ok(127.0.0.1:57272)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.691837852Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1702025-01-27T19:42:56.276Z{"msg":"ds_connection connected from Ok(127.0.0.1:55724)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.691905321Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1712025-01-27T19:42:56.276Z{"msg":"ds_connection connected from Ok(127.0.0.1:41485)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.691990126Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1722025-01-27T19:42:56.276Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.692547508Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"client":"1","":"downstairs","session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1732025-01-27T19:42:56.276Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.692628523Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"client":"2","":"downstairs","session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1742025-01-27T19:42:56.276Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.692670956Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"client":"0","":"downstairs","session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1752025-01-27T19:42:56.276Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.693309004Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"client":"0","":"downstairs","session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1762025-01-27T19:42:56.276Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.693351307Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"client":"1","":"downstairs","session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1772025-01-27T19:42:56.276Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.693437212Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"client":"2","":"downstairs","session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1782025-01-27T19:42:56.276Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.693736149Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1792025-01-27T19:42:56.276Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.69378942Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1802025-01-27T19:42:56.276Z{"msg":"[0]R flush_numbers: [4000, 4000, 4000, 4000, 4000]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.693860447Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"downstairs","":"downstairs","session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1812025-01-27T19:42:56.276Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.693896461Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"downstairs","":"downstairs","session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1822025-01-27T19:42:56.276Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.693911718Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"downstairs","":"downstairs","session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1832025-01-27T19:42:56.276Z{"msg":"[1]R flush_numbers: [4000, 4000, 4000, 4000, 4000]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.693933954Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"downstairs","":"downstairs","session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1842025-01-27T19:42:56.277Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.693947822Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"downstairs","":"downstairs","session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1852025-01-27T19:42:56.277Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.693968248Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"downstairs","":"downstairs","session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1862025-01-27T19:42:56.277Z{"msg":"[2]R flush_numbers: [4000, 4000, 4000, 4000, 4000]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.693981566Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"downstairs","":"downstairs","session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1872025-01-27T19:42:56.277Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.694001042Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"downstairs","":"downstairs","session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1882025-01-27T19:42:56.277Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.69401492Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"downstairs","":"downstairs","session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1892025-01-27T19:42:56.277Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.694034447Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"downstairs","":"downstairs","session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1902025-01-27T19:42:56.277Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.694050704Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"downstairs","":"downstairs","session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1912025-01-27T19:42:56.277Z{"msg":"Next flush: 4001","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.694081608Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"downstairs","":"downstairs","session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1922025-01-27T19:42:56.277Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.694102125Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"downstairs","":"downstairs","session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1932025-01-27T19:42:56.277Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.694115893Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1942025-01-27T19:42:56.277Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.6941285Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"client":"0","":"downstairs","session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1952025-01-27T19:42:56.277Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.694141738Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"client":"1","":"downstairs","session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1962025-01-27T19:42:56.277Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.694155575Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"client":"2","":"downstairs","session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1972025-01-27T19:42:56.277Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.694177902Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1982025-01-27T19:42:56.277Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.694213925Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
1992025-01-27T19:42:56.277Z{"msg":"3bfd5f33-f958-4404-b47b-f5fedf3ce48a is now active with session: d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.694258477Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
2002025-01-27T19:42:56.277Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.694339823Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"session_id":"d6a5f55b-89a5-45f8-a24d-b9dd61f9fcc6"}
2012025-01-27T19:42:56.277Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.694364519Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230}
2022025-01-27T19:42:56.277Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.694580821Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230}
2032025-01-27T19:42:56.277Z{"msg":"3bfd5f33-f958-4404-b47b-f5fedf3ce48a active request set","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.694607516Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2042025-01-27T19:42:56.277Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.694621704Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"client":"0","":"downstairs","session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2052025-01-27T19:42:56.277Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.694655068Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"client":"0","":"downstairs","session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2062025-01-27T19:42:56.277Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.694676454Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"client":"1","":"downstairs","session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2072025-01-27T19:42:56.277Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.694694931Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"client":"1","":"downstairs","session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2082025-01-27T19:42:56.277Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.694711238Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"client":"2","":"downstairs","session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2092025-01-27T19:42:56.277Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.694725615Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"client":"2","":"downstairs","session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2102025-01-27T19:42:56.277Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.694747562Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2112025-01-27T19:42:56.277Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.694788055Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2122025-01-27T19:42:56.277Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.694819869Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2132025-01-27T19:42:56.277Z{"msg":"ds_connection connected from Ok(127.0.0.1:56288)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.69486834Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2142025-01-27T19:42:56.277Z{"msg":"ds_connection connected from Ok(127.0.0.1:44322)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.694904604Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2152025-01-27T19:42:56.277Z{"msg":"ds_connection connected from Ok(127.0.0.1:35190)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.694932509Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2162025-01-27T19:42:56.277Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.694966823Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"client":"2","":"downstairs","session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2172025-01-27T19:42:56.277Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.694992189Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"client":"0","":"downstairs","session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2182025-01-27T19:42:56.277Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.695021723Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"client":"1","":"downstairs","session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2192025-01-27T19:42:56.281Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.695393648Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"client":"0","":"downstairs","session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2202025-01-27T19:42:56.281Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.695561169Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"client":"1","":"downstairs","session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2212025-01-27T19:42:56.281Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.695604811Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"client":"2","":"downstairs","session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2222025-01-27T19:42:56.281Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.695733229Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2232025-01-27T19:42:56.281Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.695930694Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2242025-01-27T19:42:56.281Z{"msg":"[0]R flush_numbers: [5324, 5324, 5323, 5321, 5323]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.696062441Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"downstairs","":"downstairs","session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2252025-01-27T19:42:56.281Z{"msg":"[0]R generation: [14, 14, 14, 13, 14]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.696097305Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"downstairs","":"downstairs","session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2262025-01-27T19:42:56.281Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.6961227Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"downstairs","":"downstairs","session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2272025-01-27T19:42:56.281Z{"msg":"[1]R flush_numbers: [5324, 5324, 5323, 5321, 5323]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.696152275Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"downstairs","":"downstairs","session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2282025-01-27T19:42:56.281Z{"msg":"[1]R generation: [14, 14, 14, 13, 14]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.69617897Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"downstairs","":"downstairs","session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2292025-01-27T19:42:56.281Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.696202116Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"downstairs","":"downstairs","session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2302025-01-27T19:42:56.281Z{"msg":"[2]R flush_numbers: [5324, 5324, 5323, 5321, 5323]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.696217973Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"downstairs","":"downstairs","session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2312025-01-27T19:42:56.281Z{"msg":"[2]R generation: [14, 14, 14, 13, 14]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.696251578Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"downstairs","":"downstairs","session_id":"37d9a885-5354-42c7-8e44-627fd008640a"Wait for a query_work_queue command to finish before sending IO
2322025-01-27T19:42:56.281Z}
2332025-01-27T19:42:56.281Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.696291371Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"downstairs","":"downstairs","session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2342025-01-27T19:42:56.281Z{"msg":"Max found gen is 15","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.696318836Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"downstairs","":"downstairs","session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2352025-01-27T19:42:56.281Z{"msg":"Generation requested: 31 >= found:15","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.696337842Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"downstairs","":"downstairs","session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2362025-01-27T19:42:56.281Z{"msg":"Next flush: 5325","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.696371936Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"downstairs","":"downstairs","session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2372025-01-27T19:42:56.281Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.696391143Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"":"downstairs","":"downstairs","session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2382025-01-27T19:42:56.281Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.696419018Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2392025-01-27T19:42:56.281Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.696452692Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"client":"0","":"downstairs","session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2402025-01-27T19:42:56.282Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.696476958Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"client":"1","":"downstairs","session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2412025-01-27T19:42:56.282Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.696490616Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"client":"2","":"downstairs","session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2422025-01-27T19:42:56.282Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.696511262Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2432025-01-27T19:42:56.282Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.696531548Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2442025-01-27T19:42:56.282Z{"msg":"3bfd5f33-f958-4404-b47b-f5fedf3ce48a is now active with session: 37d9a885-5354-42c7-8e44-627fd008640a","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.696565722Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2452025-01-27T19:42:56.282Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.696598757Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230,"session_id":"37d9a885-5354-42c7-8e44-627fd008640a"}
2462025-01-27T19:42:56.282Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:54.69663579Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230}
2472025-01-27T19:42:56.282ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
2482025-01-27T19:42:56.282ZFill test
2492025-01-27T19:42:56.319ZRead and Verify all blocks (0..100 range:false)
2502025-01-27T19:42:56.419ZWrote out file "/var/tmp/test_up-build/verify_file"
2512025-01-27T19:42:56.419ZCLIENT: Tests done. All submitted work has been ACK'd
2522025-01-27T19:42:56.419Z----------------------------------------------------------------
2532025-01-27T19:42:56.419Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2542025-01-27T19:42:56.419ZStates: Active Active Active
2552025-01-27T19:42:56.419ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2562025-01-27T19:42:56.422Z 1002 Acked Read 50 Done Done Done false
2572025-01-27T19:42:56.422Z STATES DS:0 DS:1 DS:2 TOTAL
2582025-01-27T19:42:56.422Z Sent 0 0 0 0
2592025-01-27T19:42:56.422Z Done 1 1 1 3
2602025-01-27T19:42:56.422Z Skipped 0 0 0 0
2612025-01-27T19:42:56.422Z Error 0 0 0 0
2622025-01-27T19:42:56.422ZLast Flush: 1001 1001 1001
2632025-01-27T19:42:56.422ZDownstairs last five completed: 1001 1000
2642025-01-27T19:42:56.422ZUpstairs last five completed: 1002 1001 1000
2652025-01-27T19:42:56.422Z----------------------------------------------------------------
2662025-01-27T19:42:56.422Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2672025-01-27T19:42:56.422ZStates: Active Active Active
2682025-01-27T19:42:56.422ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2692025-01-27T19:42:56.422Z 1002 Acked Read 50 Done Done Done false
2702025-01-27T19:42:56.422Z STATES DS:0 DS:1 DS:2 TOTAL
2712025-01-27T19:42:56.422Z Sent 0 0 0 0
2722025-01-27T19:42:56.422Z Done 1 1 1 3
2732025-01-27T19:42:56.422Z Skipped 0 0 0 0
2742025-01-27T19:42:56.422Z Error 0 0 0 0
2752025-01-27T19:42:56.422ZLast Flush: 1001 1001 1001
2762025-01-27T19:42:56.422ZDownstairs last five completed: 1001 1000
2772025-01-27T19:42:56.422ZUpstairs last five completed: 1002 1001 1000
2782025-01-27T19:42:56.422ZCLIENT: Up:0 ds:2 act:6
2792025-01-27T19:43:00.421Z----------------------------------------------------------------
2802025-01-27T19:43:00.421Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2812025-01-27T19:43:00.421ZDownstairs last five completed: 1003 1002 1001 1000
2822025-01-27T19:43:00.421ZUpstairs last five completed: 1003 1002 1001 1000
2832025-01-27T19:43:00.421Z----------------------------------------------------------------
2842025-01-27T19:43:00.421Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2852025-01-27T19:43:00.421ZDownstairs last five completed: 1003 1002 1001 1000
2862025-01-27T19:43:00.421ZUpstairs last five completed: 1003 1002 1001 1000
2872025-01-27T19:43:00.425ZCLIENT: Up:0 ds:0 act:6
2882025-01-27T19:43:00.425ZCLIENT: All crucible jobs finished, exiting program
2892025-01-27T19:43:00.425Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-27T19:42:58.840191906Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1230}
2902025-01-27T19:43:00.432ZRepair setup passed
2912025-01-27T19:43:00.436ZCopy the region for /var/tmp/test_up-build/8830
2922025-01-27T19:43:00.439Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2932025-01-27T19:43:00.441Z/var/tmp/bins/crutest fill -g 32 -q --key XIsiXvqqCpigCMpSEKjd+Elr+bwxGR3bvpkVLYo+Vjs= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2942025-01-27T19:43:00.498Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.917437486Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"task":"crutest"}
2952025-01-27T19:43:00.501Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.918032762Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"task":"crutest"}
2962025-01-27T19:43:00.501Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.919734063Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"task":"crutest"}
2972025-01-27T19:43:00.501Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.919946416Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236}
2982025-01-27T19:43:00.501Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d77aa90b6afe7d7ddb1dc63211fa4216c8866a90\",\n git_commit_timestamp: \"2025-01-27T19:20:16.000000000Z\",\n git_branch: \"mkeeter/shrink-replay-buffer\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.919992478Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236}
2992025-01-27T19:43:00.501Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.920016894Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236}
3002025-01-27T19:43:00.501Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.920032311Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236}
3012025-01-27T19:43:00.502Z{"msg":"Crucible 4bbdee8a-8165-49d4-9f53-e88a31538e91 has session id: 2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.920656101Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3022025-01-27T19:43:00.504Z{"msg":"Upstairs opts: Upstairs UUID: 4bbdee8a-8165-49d4-9f53-e88a31538e91, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.920747095Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3032025-01-27T19:43:00.505Z{"msg":"Crucible stats registered with UUID: 4bbdee8a-8165-49d4-9f53-e88a31538e91","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.920785019Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3042025-01-27T19:43:00.505Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-27T19:42:58.920808535Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3052025-01-27T19:43:00.505Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.922911835Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"task":"crutest"}
3062025-01-27T19:43:00.505Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.923006649Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236}
3072025-01-27T19:43:00.505Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d77aa90b6afe7d7ddb1dc63211fa4216c8866a90\",\n git_commit_timestamp: \"2025-01-27T19:20:16.000000000Z\",\n git_branch: \"mkeeter/shrink-replay-buffer\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.923043572Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236}
3082025-01-27T19:43:00.505Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.92305646Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236}
3092025-01-27T19:43:00.505Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.923076676Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236}
3102025-01-27T19:43:00.505Z{"msg":"Crucible 4bbdee8a-8165-49d4-9f53-e88a31538e91 has session id: a467fe6a-8eef-4d88-8454-0044ec4b16bc","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.923446081Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3112025-01-27T19:43:00.505Z{"msg":"Upstairs opts: Upstairs UUID: 4bbdee8a-8165-49d4-9f53-e88a31538e91, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.923493923Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3122025-01-27T19:43:00.505Z{"msg":"Crucible stats registered with UUID: 4bbdee8a-8165-49d4-9f53-e88a31538e91","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.923517819Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3132025-01-27T19:43:00.505Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-27T19:42:58.923548363Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3142025-01-27T19:43:00.508Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.924227164Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"task":"crutest"}
3152025-01-27T19:43:00.508Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.924274986Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236}
3162025-01-27T19:43:00.508Z{"msg":"4bbdee8a-8165-49d4-9f53-e88a31538e91 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.924319298Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3172025-01-27T19:43:00.508Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.924345034Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"client":"0","":"downstairs","session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3182025-01-27T19:43:00.508Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.924378138Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"client":"0","":"downstairs","session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3192025-01-27T19:43:00.508Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.924394395Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"client":"1","":"downstairs","session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3202025-01-27T19:43:00.508Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.92442385Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"client":"1","":"downstairs","session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3212025-01-27T19:43:00.508Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.924463993Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"client":"2","":"downstairs","session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3222025-01-27T19:43:00.508Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.924507785Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"client":"2","":"downstairs","session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3232025-01-27T19:43:00.508Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.924546548Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"io task","client":"2","":"downstairs","session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3242025-01-27T19:43:00.508Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.924579282Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"io task","client":"0","":"downstairs","session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3252025-01-27T19:43:00.508Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.924618586Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"io task","client":"1","":"downstairs","session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3262025-01-27T19:43:00.508Z{"msg":"ds_connection connected from Ok(127.0.0.1:34031)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.924647401Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"io task","client":"0","":"downstairs","session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3272025-01-27T19:43:00.508Z{"msg":"ds_connection connected from Ok(127.0.0.1:51301)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.924672096Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"io task","client":"2","":"downstairs","session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3282025-01-27T19:43:00.508Z{"msg":"ds_connection connected from Ok(127.0.0.1:64605)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.924838607Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"io task","client":"1","":"downstairs","session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3292025-01-27T19:43:00.508Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.925340799Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"client":"2","":"downstairs","session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3302025-01-27T19:43:00.509Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.925385021Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"client":"0","":"downstairs","session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3312025-01-27T19:43:00.509Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.925411396Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"client":"1","":"downstairs","session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3322025-01-27T19:43:00.509Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.926034027Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"client":"1","":"downstairs","session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3332025-01-27T19:43:00.509Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.926096336Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"client":"2","":"downstairs","session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3342025-01-27T19:43:00.509Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.926136469Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"client":"0","":"downstairs","session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3352025-01-27T19:43:00.509Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.926459592Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3362025-01-27T19:43:00.509Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.926493366Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3372025-01-27T19:43:00.509Z{"msg":"[0]R flush_numbers: [4001, 4001, 4001, 4001, 4001]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.926508004Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"downstairs","":"downstairs","session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3382025-01-27T19:43:00.509Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.926524621Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"downstairs","":"downstairs","session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3392025-01-27T19:43:00.509Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.926547227Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"downstairs","":"downstairs","session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3402025-01-27T19:43:00.509Z{"msg":"[1]R flush_numbers: [4001, 4001, 4001, 4001, 4001]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.926564624Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"downstairs","":"downstairs","session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3412025-01-27T19:43:00.509Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.926596048Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"downstairs","":"downstairs","session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3422025-01-27T19:43:00.509Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.926611376Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"downstairs","":"downstairs","session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3432025-01-27T19:43:00.509Z{"msg":"[2]R flush_numbers: [4001, 4001, 4001, 4001, 4001]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.9266417Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"downstairs","":"downstairs","session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3442025-01-27T19:43:00.509Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.926658957Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"downstairs","":"downstairs","session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3452025-01-27T19:43:00.509Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.926690692Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"downstairs","":"downstairs","session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3462025-01-27T19:43:00.509Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.926705069Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"downstairs","":"downstairs","session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3472025-01-27T19:43:00.509Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.926722886Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"downstairs","":"downstairs","session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3482025-01-27T19:43:00.509Z{"msg":"Next flush: 4002","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.92675572Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"downstairs","":"downstairs","session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3492025-01-27T19:43:00.509Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.926780946Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"downstairs","":"downstairs","session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3502025-01-27T19:43:00.509Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.92681352Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3512025-01-27T19:43:00.509Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.926836996Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"client":"0","":"downstairs","session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3522025-01-27T19:43:00.509Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.926877209Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"client":"1","":"downstairs","session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3532025-01-27T19:43:00.509Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.926913763Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"client":"2","":"downstairs","session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3542025-01-27T19:43:00.509Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.927126446Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3552025-01-27T19:43:00.509Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.92715595Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3562025-01-27T19:43:00.509Z{"msg":"4bbdee8a-8165-49d4-9f53-e88a31538e91 is now active with session: 2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.927180076Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3572025-01-27T19:43:00.509Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.927203662Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"session_id":"2cc0c5a2-c0b4-4ddd-bac4-f0447b3ed21c"}
3582025-01-27T19:43:00.509Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.927228328Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236}
3592025-01-27T19:43:00.509Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.927263411Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236}
3602025-01-27T19:43:00.512Z{"msg":"4bbdee8a-8165-49d4-9f53-e88a31538e91 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.927287777Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3612025-01-27T19:43:00.512Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.927318852Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"client":"0","":"downstairs","session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3622025-01-27T19:43:00.513Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.927421564Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"client":"0","":"downstairs","session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3632025-01-27T19:43:00.513Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.927455138Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"client":"1","":"downstairs","session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3642025-01-27T19:43:00.513Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.927511958Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"client":"1","":"downstairs","session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3652025-01-27T19:43:00.513Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.92755419Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"client":"2","":"downstairs","session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3662025-01-27T19:43:00.513Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.927578476Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"client":"2","":"downstairs","session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3672025-01-27T19:43:00.513Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.927640295Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"io task","client":"0","":"downstairs","session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3682025-01-27T19:43:00.513Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.927678209Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"io task","client":"2","":"downstairs","session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3692025-01-27T19:43:00.513Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.927715802Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"io task","client":"1","":"downstairs","session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3702025-01-27T19:43:00.513Z{"msg":"ds_connection connected from Ok(127.0.0.1:40897)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.927760274Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"io task","client":"2","":"downstairs","session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3712025-01-27T19:43:00.513Z{"msg":"ds_connection connected from Ok(127.0.0.1:58369)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.927799817Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"io task","client":"0","":"downstairs","session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3722025-01-27T19:43:00.513Z{"msg":"ds_connection connected from Ok(127.0.0.1:64454)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.927831951Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"io task","client":"1","":"downstairs","session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3732025-01-27T19:43:00.513Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.927870825Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"client":"0","":"downstairs","session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3742025-01-27T19:43:00.513Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.927912157Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"client":"2","":"downstairs","session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3752025-01-27T19:43:00.513Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.927934664Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"client":"1","":"downstairs","session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3762025-01-27T19:43:00.513Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.927972847Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"client":"2","":"downstairs","session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3772025-01-27T19:43:00.513Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.928155845Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"client":"1","":"downstairs","session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3782025-01-27T19:43:00.513Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.928273464Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"client":"0","":"downstairs","session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3792025-01-27T19:43:00.513Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.928384015Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3802025-01-27T19:43:00.513Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.928473979Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3812025-01-27T19:43:00.513Z{"msg":"[0]R flush_numbers: [5325, 5325, 5325, 5325, 5325]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.928568912Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"downstairs","":"downstairs","session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3822025-01-27T19:43:00.513Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.928595377Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"downstairs","":"downstairs","session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3832025-01-27T19:43:00.513Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.928617234Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"downstairs","":"downstairs","session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3842025-01-27T19:43:00.513Z{"msg":"[1]R flush_numbers: [5325, 5325, 5325, 5325, 5325]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.928656167Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"downstairs","":"downstairs","session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3852025-01-27T19:43:00.513Z{Wait for a query_work_queue command to finish before sending IO
3862025-01-27T19:43:00.513Z"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.928706018Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"downstairs","":"downstairs","session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3872025-01-27T19:43:00.513Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.928742262Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"downstairs","":"downstairs","session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3882025-01-27T19:43:00.513Z{"msg":"[2]R flush_numbers: [5325, 5325, 5325, 5325, 5325]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.928758719Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"downstairs","":"downstairs","session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3892025-01-27T19:43:00.513Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.928771846Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"downstairs","":"downstairs","session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3902025-01-27T19:43:00.513Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.928785224Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"downstairs","":"downstairs","session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3912025-01-27T19:43:00.513Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.928817649Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"downstairs","":"downstairs","session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3922025-01-27T19:43:00.513Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.928843664Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"downstairs","":"downstairs","session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3932025-01-27T19:43:00.513Z{"msg":"Next flush: 5326","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.928869899Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"downstairs","":"downstairs","session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3942025-01-27T19:43:00.513Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.928883607Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"":"downstairs","":"downstairs","session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3952025-01-27T19:43:00.513Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.928903363Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3962025-01-27T19:43:00.513Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.92892479Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"client":"0","":"downstairs","session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3972025-01-27T19:43:00.513Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.928957824Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"client":"1","":"downstairs","session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3982025-01-27T19:43:00.513Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.928973061Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"client":"2","":"downstairs","session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
3992025-01-27T19:43:00.513Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.928986779Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
4002025-01-27T19:43:00.513Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.929041449Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
4012025-01-27T19:43:00.514Z{"msg":"4bbdee8a-8165-49d4-9f53-e88a31538e91 is now active with session: a467fe6a-8eef-4d88-8454-0044ec4b16bc","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.929071024Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
4022025-01-27T19:43:00.514Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.929088621Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236,"session_id":"a467fe6a-8eef-4d88-8454-0044ec4b16bc"}
4032025-01-27T19:43:00.514Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:42:58.929111197Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236}
4042025-01-27T19:43:00.514ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
4052025-01-27T19:43:00.514ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4062025-01-27T19:43:00.514ZFill test
4072025-01-27T19:43:00.552ZRead and Verify all blocks (0..100 range:false)
4082025-01-27T19:43:00.630ZWrote out file "/var/tmp/test_up-build/verify_file"
4092025-01-27T19:43:00.630ZCLIENT: Tests done. All submitted work has been ACK'd
4102025-01-27T19:43:00.633Z----------------------------------------------------------------
4112025-01-27T19:43:00.633Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4122025-01-27T19:43:00.633ZStates: Active Active Active
4132025-01-27T19:43:00.633ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4142025-01-27T19:43:00.633Z 1002 Acked Read 50 Done Done Done false
4152025-01-27T19:43:00.633Z STATES DS:0 DS:1 DS:2 TOTAL
4162025-01-27T19:43:00.634Z Sent 0 0 0 0
4172025-01-27T19:43:00.634Z Done 1 1 1 3
4182025-01-27T19:43:00.634Z Skipped 0 0 0 0
4192025-01-27T19:43:00.634Z Error 0 0 0 0
4202025-01-27T19:43:00.634ZLast Flush: 1001 1001 1001
4212025-01-27T19:43:00.634ZDownstairs last five completed: 1001 1000
4222025-01-27T19:43:00.634ZUpstairs last five completed: 1002 1001 1000
4232025-01-27T19:43:00.634Z----------------------------------------------------------------
4242025-01-27T19:43:00.634Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4252025-01-27T19:43:00.634ZStates: Active Active Active
4262025-01-27T19:43:00.634ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4272025-01-27T19:43:00.634Z 1002 Acked Read 50 Done Done Done false
4282025-01-27T19:43:00.634Z STATES DS:0 DS:1 DS:2 TOTAL
4292025-01-27T19:43:00.634Z Sent 0 0 0 0
4302025-01-27T19:43:00.634Z Done 1 1 1 3
4312025-01-27T19:43:00.634Z Skipped 0 0 0 0
4322025-01-27T19:43:00.634Z Error 0 0 0 0
4332025-01-27T19:43:00.634ZLast Flush: 1001 1001 1001
4342025-01-27T19:43:00.634ZDownstairs last five completed: 1001 1000
4352025-01-27T19:43:00.634ZUpstairs last five completed: 1002 1001 1000
4362025-01-27T19:43:00.634ZCLIENT: Up:0 ds:2 act:6
4372025-01-27T19:43:04.633Z----------------------------------------------------------------
4382025-01-27T19:43:04.633Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4392025-01-27T19:43:04.633ZDownstairs last five completed: 1003 1002 1001 1000
4402025-01-27T19:43:04.633ZUpstairs last five completed: 1003 1002 1001 1000
4412025-01-27T19:43:04.633Z----------------------------------------------------------------
4422025-01-27T19:43:04.633Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4432025-01-27T19:43:04.633ZDownstairs last five completed: 1003 1002 1001 1000
4442025-01-27T19:43:04.633ZUpstairs last five completed: 1003 1002 1001 1000
4452025-01-27T19:43:04.637ZCLIENT: Up:0 ds:0 act:6
4462025-01-27T19:43:04.637ZCLIENT: All crucible jobs finished, exiting program
4472025-01-27T19:43:04.637Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-27T19:43:03.051696205Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1236}
4482025-01-27T19:43:04.640ZRepair part 1 passed
4492025-01-27T19:43:04.643Z
4502025-01-27T19:43:04.643ZKill the current downstairs
4512025-01-27T19:43:04.814ZDownstairs 2 stopped
4522025-01-27T19:43:04.814Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
4532025-01-27T19:43:04.818ZNow put back the original so we have a mismatch
4542025-01-27T19:43:04.818Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
4552025-01-27T19:43:04.820ZRestart downstairs with old directory
4562025-01-27T19:43:04.877Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4572025-01-27T19:43:04.895ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4582025-01-27T19:43:04.898Z 0 000-009 32 32 31 4002 4002 4001 F F F <---
4592025-01-27T19:43:04.898Z 1 010-019 32 32 31 4002 4002 4001 F F F <---
4602025-01-27T19:43:04.898Z 2 020-029 32 32 31 4002 4002 4001 F F F <---
4612025-01-27T19:43:04.898Z 3 030-039 32 32 31 4002 4002 4001 F F F <---
4622025-01-27T19:43:04.898Z 4 040-049 32 32 31 4002 4002 4001 F F F <---
4632025-01-27T19:43:04.898ZMax gen: 32, Max flush: 4002
4642025-01-27T19:43:04.898ZError: Difference in extent metadata found!
4652025-01-27T19:43:04.901Zdump test found error as expected
4662025-01-27T19:43:04.901Z
4672025-01-27T19:43:04.901Z
4682025-01-27T19:43:04.901Z/var/tmp/bins/crutest verify --range -g 33 -q --key XIsiXvqqCpigCMpSEKjd+Elr+bwxGR3bvpkVLYo+Vjs= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
4692025-01-27T19:43:04.962Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.378795166Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"task":"crutest"}
4702025-01-27T19:43:04.965Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.379469787Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"task":"crutest"}
4712025-01-27T19:43:04.965Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.381097632Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"task":"crutest"}
4722025-01-27T19:43:04.965Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.381302765Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244}
4732025-01-27T19:43:04.965Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d77aa90b6afe7d7ddb1dc63211fa4216c8866a90\",\n git_commit_timestamp: \"2025-01-27T19:20:16.000000000Z\",\n git_branch: \"mkeeter/shrink-replay-buffer\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.381347428Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244}
4742025-01-27T19:43:04.965Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.381362925Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244}
4752025-01-27T19:43:04.965Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.381377442Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244}
4762025-01-27T19:43:04.965Z{"msg":"Crucible 53265cbb-34c8-4b7d-b2d7-1d26420bd9b1 has session id: c11d7c8a-2a24-4546-8aa9-b5c97c205955","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.381948852Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
4772025-01-27T19:43:04.965Z{"msg":"Upstairs opts: Upstairs UUID: 53265cbb-34c8-4b7d-b2d7-1d26420bd9b1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.382000243Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
4782025-01-27T19:43:04.965Z{"msg":"Crucible stats registered with UUID: 53265cbb-34c8-4b7d-b2d7-1d26420bd9b1","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.3820182Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
4792025-01-27T19:43:04.965Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-27T19:43:03.382032108Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
4802025-01-27T19:43:04.969Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.384152105Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"task":"crutest"}
4812025-01-27T19:43:04.969Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.38424025Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244}
4822025-01-27T19:43:04.969Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d77aa90b6afe7d7ddb1dc63211fa4216c8866a90\",\n git_commit_timestamp: \"2025-01-27T19:20:16.000000000Z\",\n git_branch: \"mkeeter/shrink-replay-buffer\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.384282962Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244}
4832025-01-27T19:43:04.969Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.384303609Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244}
4842025-01-27T19:43:04.969Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.384318776Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244}
4852025-01-27T19:43:04.969Z{"msg":"Crucible 53265cbb-34c8-4b7d-b2d7-1d26420bd9b1 has session id: d7d167c3-0076-4ef7-91c0-256fa5a35a11","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.384740692Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
4862025-01-27T19:43:04.969Z{"msg":"Upstairs opts: Upstairs UUID: 53265cbb-34c8-4b7d-b2d7-1d26420bd9b1, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.384782325Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
4872025-01-27T19:43:04.969Z{"msg":"Crucible stats registered with UUID: 53265cbb-34c8-4b7d-b2d7-1d26420bd9b1","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.384798342Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
4882025-01-27T19:43:04.969Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-27T19:43:03.384817788Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
4892025-01-27T19:43:04.969Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.38554764Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"task":"crutest"}
4902025-01-27T19:43:04.969Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.385584094Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244}
4912025-01-27T19:43:04.969Z{"msg":"53265cbb-34c8-4b7d-b2d7-1d26420bd9b1 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.385672418Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
4922025-01-27T19:43:04.969Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.385699604Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"0","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
4932025-01-27T19:43:04.969Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.385727629Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"0","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
4942025-01-27T19:43:04.969Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.385766852Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"1","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
4952025-01-27T19:43:04.969Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.385791697Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"1","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
4962025-01-27T19:43:04.969Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.385830081Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"2","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
4972025-01-27T19:43:04.969Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.385861025Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"2","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
4982025-01-27T19:43:04.969Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.38589144Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
4992025-01-27T19:43:04.969Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.385917215Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5002025-01-27T19:43:04.969Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.385941741Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5012025-01-27T19:43:04.969Z{"msg":"ds_connection connected from Ok(127.0.0.1:63609)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.385967037Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5022025-01-27T19:43:04.969Z{"msg":"ds_connection connected from Ok(127.0.0.1:42694)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.386000181Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5032025-01-27T19:43:04.969Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-01-27T19:43:03.386024177Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5042025-01-27T19:43:04.969Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-01-27T19:43:03.386056661Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5052025-01-27T19:43:04.969Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-01-27T19:43:03.386083886Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5062025-01-27T19:43:04.969Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.386163042Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5072025-01-27T19:43:04.969Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.386624161Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"0","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5082025-01-27T19:43:04.969Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.386791662Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"1","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5092025-01-27T19:43:04.973Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.387166096Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"0","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5102025-01-27T19:43:04.973Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.387292034Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"1","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5112025-01-27T19:43:04.973Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.387511826Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5122025-01-27T19:43:04.973Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:03.387592201Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5132025-01-27T19:43:14.973Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.387982794Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5142025-01-27T19:43:14.973Z{"msg":"ds_connection connected from Ok(127.0.0.1:55854)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.388240049Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5152025-01-27T19:43:14.983Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.389668978Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"2","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5162025-01-27T19:43:14.984Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.390235488Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"2","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5172025-01-27T19:43:14.984Z{"msg":"[0]R flush_numbers: [4002, 4002, 4002, 4002, 4002]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.390633318Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"downstairs","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5182025-01-27T19:43:14.984Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.390663973Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"downstairs","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5192025-01-27T19:43:14.984Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.39068017Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"downstairs","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5202025-01-27T19:43:14.984Z{"msg":"[1]R flush_numbers: [4002, 4002, 4002, 4002, 4002]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.390712875Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"downstairs","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5212025-01-27T19:43:14.984Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.390742899Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"downstairs","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5222025-01-27T19:43:14.984Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.390775973Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"downstairs","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5232025-01-27T19:43:14.984Z{"msg":"[2]R flush_numbers: [4001, 4001, 4001, 4001, 4001]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.390809987Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"downstairs","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5242025-01-27T19:43:14.984Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.390843092Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"downstairs","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5252025-01-27T19:43:14.984Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.390874476Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"downstairs","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5262025-01-27T19:43:14.984Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.390890883Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"downstairs","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5272025-01-27T19:43:14.984Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.390920858Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"downstairs","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5282025-01-27T19:43:14.984Z{"msg":"Next flush: 4003","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.390938885Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"downstairs","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5292025-01-27T19:43:14.984Z{"msg":"extent 0 needs reconciliation: ExtentMetadata { gen: 32, flush: 4002, dirty: false } ExtentMetadata { gen: 32, flush: 4002, dirty: false } ExtentMetadata { gen: 31, flush: 4001, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.390970729Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5302025-01-27T19:43:14.984Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 4002, dirty: false }, ExtentMetadata { gen: 32, flush: 4002, dirty: false }, ExtentMetadata { gen: 31, flush: 4001, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.390989206Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5312025-01-27T19:43:14.984Z{"msg":"found dest for source 0 for extent at index 0 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.391008772Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5322025-01-27T19:43:14.984Z{"msg":"extent 1 needs reconciliation: ExtentMetadata { gen: 32, flush: 4002, dirty: false } ExtentMetadata { gen: 32, flush: 4002, dirty: false } ExtentMetadata { gen: 31, flush: 4001, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.391035538Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5332025-01-27T19:43:14.984Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 4002, dirty: false }, ExtentMetadata { gen: 32, flush: 4002, dirty: false }, ExtentMetadata { gen: 31, flush: 4001, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.391061583Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5342025-01-27T19:43:14.984Z{"msg":"found dest for source 0 for extent at index 1 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.391092428Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5352025-01-27T19:43:14.984Z{"msg":"extent 2 needs reconciliation: ExtentMetadata { gen: 32, flush: 4002, dirty: false } ExtentMetadata { gen: 32, flush: 4002, dirty: false } ExtentMetadata { gen: 31, flush: 4001, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.391108925Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5362025-01-27T19:43:14.984Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 4002, dirty: false }, ExtentMetadata { gen: 32, flush: 4002, dirty: false }, ExtentMetadata { gen: 31, flush: 4001, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.391141769Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5372025-01-27T19:43:14.984Z{"msg":"found dest for source 0 for extent at index 2 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.391174183Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5382025-01-27T19:43:14.984Z{"msg":"extent 3 needs reconciliation: ExtentMetadata { gen: 32, flush: 4002, dirty: false } ExtentMetadata { gen: 32, flush: 4002, dirty: false } ExtentMetadata { gen: 31, flush: 4001, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.391200809Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5392025-01-27T19:43:14.984Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 4002, dirty: false }, ExtentMetadata { gen: 32, flush: 4002, dirty: false }, ExtentMetadata { gen: 31, flush: 4001, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.391229934Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5402025-01-27T19:43:14.984Z{"msg":"found dest for source 0 for extent at index 3 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.391271177Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5412025-01-27T19:43:14.984Z{"msg":"extent 4 needs reconciliation: ExtentMetadata { gen: 32, flush: 4002, dirty: false } ExtentMetadata { gen: 32, flush: 4002, dirty: false } ExtentMetadata { gen: 31, flush: 4001, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.391305351Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5422025-01-27T19:43:14.984Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 4002, dirty: false }, ExtentMetadata { gen: 32, flush: 4002, dirty: false }, ExtentMetadata { gen: 31, flush: 4001, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.391349543Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5432025-01-27T19:43:14.984Z{"msg":"found dest for source 0 for extent at index 4 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.391391056Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"mend","":"downstairs","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5442025-01-27T19:43:14.984Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.391417711Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"0","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5452025-01-27T19:43:14.985Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.391432488Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"1","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5462025-01-27T19:43:14.985Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.391454154Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"2","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5472025-01-27T19:43:14.985Z{"msg":"Full repair list: {ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.39147692Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"downstairs","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5482025-01-27T19:43:14.985Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 4003, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 4003, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 4003, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 4003, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 4003, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.391572833Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"downstairs","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5492025-01-27T19:43:14.985Z{"msg":"starting reconciliation 993cc27e-bfeb-4070-a1f3-52ddd43fc30f: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.39159237Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"downstairs","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5502025-01-27T19:43:14.985Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.391605808Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"2","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5512025-01-27T19:43:14.988Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.400294742Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"2","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5522025-01-27T19:43:14.988Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.404099135Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"2","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5532025-01-27T19:43:14.991Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.407808463Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"2","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5542025-01-27T19:43:14.995Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.411344053Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"2","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5552025-01-27T19:43:14.998Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.414420623Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"downstairs","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5562025-01-27T19:43:14.998Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.414449008Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"0","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5572025-01-27T19:43:14.998Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.414464305Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"1","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5582025-01-27T19:43:14.998Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.414477623Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"2","":"downstairs","session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5592025-01-27T19:43:14.998Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.41449073Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5602025-01-27T19:43:14.998Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.414504608Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5612025-01-27T19:43:14.998Z{"msg":"53265cbb-34c8-4b7d-b2d7-1d26420bd9b1 is now active with session: c11d7c8a-2a24-4546-8aa9-b5c97c205955","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.414517956Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"session_id":"c11d7c8a-2a24-4546-8aa9-b5c97c205955"}
5622025-01-27T19:43:14.998Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.414534083Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244}
5632025-01-27T19:43:15.002Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.414585064Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244}
5642025-01-27T19:43:15.002Z{"msg":"53265cbb-34c8-4b7d-b2d7-1d26420bd9b1 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.414605061Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
5652025-01-27T19:43:15.002Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.414618378Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"0","":"downstairs","session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
5662025-01-27T19:43:15.002Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.414647413Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"0","":"downstairs","session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
5672025-01-27T19:43:15.002Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.414680177Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"1","":"downstairs","session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
5682025-01-27T19:43:15.002Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.414711562Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"1","":"downstairs","session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
5692025-01-27T19:43:15.002Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.414739057Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"2","":"downstairs","session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
5702025-01-27T19:43:15.002Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.414757224Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"2","":"downstairs","session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
5712025-01-27T19:43:15.002Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.41478044Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
5722025-01-27T19:43:15.002Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.414815004Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
5732025-01-27T19:43:15.002Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.414841109Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
5742025-01-27T19:43:15.002Z{"msg":"ds_connection connected from Ok(127.0.0.1:61623)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.414865475Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
5752025-01-27T19:43:15.002Z{"msg":"ds_connection connected from Ok(127.0.0.1:46879)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.41489068Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
5762025-01-27T19:43:15.002Z{"msg":"ds_connection connected from Ok(127.0.0.1:50597)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.414915066Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
5772025-01-27T19:43:15.002Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.415259046Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"0","":"downstairs","session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
5782025-01-27T19:43:15.002Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.415300828Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"2","":"downstairs","session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
5792025-01-27T19:43:15.002Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.415328394Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"1","":"downstairs","session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
5802025-01-27T19:43:15.002Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.415845543Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"0","":"downstairs","session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
5812025-01-27T19:43:15.002Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.415887015Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"1","":"downstairs","session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
5822025-01-27T19:43:15.002Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.416035639Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"2","":"downstairs","session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
5832025-01-27T19:43:15.002Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.416238374Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
5842025-01-27T19:43:15.002Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.416273868Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
5852025-01-27T19:43:15.002Z{"msg":"[0]R flush_numbers: [5326, 5326, 5326, 5326, 5326]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.416355023Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"downstairs","":"downstairs","session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
5862025-01-27T19:43:15.002Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.416382458Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"downstairs","":"downstairs","session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
5872025-01-27T19:43:15.002Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.416397256Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"downstairs","":"downstairs","session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
5882025-01-27T19:43:15.002Z{"msg":"[1]R flush_numbers: [5326, 5326, 5326, 5326, 5326]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.416417942Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"downstairs","":"downstairs","session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
5892025-01-27T19:43:15.002Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.41643128Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"downstairs","":"downstairs","session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
5902025-01-27T19:43:15.002Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.416451316Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"downstairs","":"downstairs","session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
5912025-01-27T19:43:15.002Z{"msg":"Wait for a query_work_queue command to finish before sending IO
5922025-01-27T19:43:15.002Z[2]R flush_numbers: [5326, 5326, 5326, 5326, 5326]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.416508966Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"downstairs","":"downstairs","session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
5932025-01-27T19:43:15.002Z{"msg":"[2]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.416531682Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"downstairs","":"downstairs","session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
5942025-01-27T19:43:15.003Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.41654515Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"downstairs","":"downstairs","session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
5952025-01-27T19:43:15.003Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.416564737Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"downstairs","":"downstairs","session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
5962025-01-27T19:43:15.003Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.416577654Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"downstairs","":"downstairs","session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
5972025-01-27T19:43:15.003Z{"msg":"Next flush: 5327","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.416590162Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"downstairs","":"downstairs","session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
5982025-01-27T19:43:15.003Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.41660263Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"":"downstairs","":"downstairs","session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
5992025-01-27T19:43:15.003Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.416615677Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
6002025-01-27T19:43:15.003Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.416641463Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"0","":"downstairs","session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
6012025-01-27T19:43:15.003Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.416662789Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"1","":"downstairs","session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
6022025-01-27T19:43:15.003Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.416676057Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"client":"2","":"downstairs","session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
6032025-01-27T19:43:15.003Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.416695754Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
6042025-01-27T19:43:15.003Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.416708021Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
6052025-01-27T19:43:15.003Z{"msg":"53265cbb-34c8-4b7d-b2d7-1d26420bd9b1 is now active with session: d7d167c3-0076-4ef7-91c0-256fa5a35a11","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.416726978Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
6062025-01-27T19:43:15.003Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.416739126Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244,"session_id":"d7d167c3-0076-4ef7-91c0-256fa5a35a11"}
6072025-01-27T19:43:15.003Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:43:13.416757652Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244}
6082025-01-27T19:43:15.003ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
6092025-01-27T19:43:15.003ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
6102025-01-27T19:43:15.003ZRead and Verify all blocks (0..100 range:true)
6112025-01-27T19:43:15.080ZWrote out file "/var/tmp/test_up-build/verify_file"
6122025-01-27T19:43:15.080ZVerify test completed
6132025-01-27T19:43:15.084ZWrote out file "/var/tmp/test_up-build/verify_file"
6142025-01-27T19:43:15.084ZCLIENT: Tests done. All submitted work has been ACK'd
6152025-01-27T19:43:15.084Z----------------------------------------------------------------
6162025-01-27T19:43:15.084Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6172025-01-27T19:43:15.084ZStates: Active Active Active
6182025-01-27T19:43:15.084ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6192025-01-27T19:43:15.084Z 1000 Acked Read 50 Done Done Done false
6202025-01-27T19:43:15.084Z STATES DS:0 DS:1 DS:2 TOTAL
6212025-01-27T19:43:15.084Z Sent 0 0 0 0
6222025-01-27T19:43:15.084Z Done 1 1 1 3
6232025-01-27T19:43:15.084Z Skipped 0 0 0 0
6242025-01-27T19:43:15.084Z Error 0 0 0 0
6252025-01-27T19:43:15.084ZLast Flush: 0 0 0
6262025-01-27T19:43:15.084ZDownstairs last five completed:
6272025-01-27T19:43:15.084ZUpstairs last five completed: 1000
6282025-01-27T19:43:15.084Z----------------------------------------------------------------
6292025-01-27T19:43:15.084Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6302025-01-27T19:43:15.084ZStates: Active Active Active
6312025-01-27T19:43:15.084ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6322025-01-27T19:43:15.084Z 1000 Acked Read 50 Sent Done Sent false
6332025-01-27T19:43:15.084Z STATES DS:0 DS:1 DS:2 TOTAL
6342025-01-27T19:43:15.084Z Sent 1 0 1 2
6352025-01-27T19:43:15.084Z Done 0 1 0 1
6362025-01-27T19:43:15.084Z Skipped 0 0 0 0
6372025-01-27T19:43:15.085Z Error 0 0 0 0
6382025-01-27T19:43:15.085ZLast Flush: 0 0 0
6392025-01-27T19:43:15.085ZDownstairs last five completed:
6402025-01-27T19:43:15.085ZUpstairs last five completed: 1000
6412025-01-27T19:43:15.085ZCLIENT: Up:0 ds:2 act:6
6422025-01-27T19:43:19.083Z----------------------------------------------------------------
6432025-01-27T19:43:19.083Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6442025-01-27T19:43:19.083ZDownstairs last five completed: 1001 1000
6452025-01-27T19:43:19.083ZUpstairs last five completed: 1001 1000
6462025-01-27T19:43:19.086Z----------------------------------------------------------------
6472025-01-27T19:43:19.086Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6482025-01-27T19:43:19.086ZDownstairs last five completed: 1001 1000
6492025-01-27T19:43:19.086ZUpstairs last five completed: 1001 1000
6502025-01-27T19:43:19.086ZCLIENT: Up:0 ds:0 act:6
6512025-01-27T19:43:19.086ZCLIENT: All crucible jobs finished, exiting program
6522025-01-27T19:43:19.087Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-27T19:43:17.499085317Z","hostname":"w-01JJMM53SJ2DSW4RAZMQ3Q0NNR","pid":1244}
6532025-01-27T19:43:19.090ZRepair part 2 passed
6542025-01-27T19:43:19.093Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
6552025-01-27T19:43:19.109ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6562025-01-27T19:43:19.109Z 0 000-009 32 32 32 4002 4002 4002 F F F
6572025-01-27T19:43:19.109Z 1 010-019 32 32 32 4002 4002 4002 F F F
6582025-01-27T19:43:19.109Z 2 020-029 32 32 32 4002 4002 4002 F F F
6592025-01-27T19:43:19.112Z 3 030-039 32 32 32 4002 4002 4002 F F F
6602025-01-27T19:43:19.112Z 4 040-049 32 32 32 4002 4002 4002 F F F
6612025-01-27T19:43:19.112ZMax gen: 32, Max flush: 4002
6622025-01-27T19:43:19.115Zdump test passed
6632025-01-27T19:43:19.115Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
6642025-01-27T19:43:19.131Z Extent 1
6652025-01-27T19:43:19.134ZGEN 32 32 32
6662025-01-27T19:43:19.134ZFLUSH_ID 4002 4002 4002
6672025-01-27T19:43:19.134ZDIRTY
6682025-01-27T19:43:19.134Z
6692025-01-27T19:43:19.134ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
6702025-01-27T19:43:19.134Z010 A A A A A A
6712025-01-27T19:43:19.137Z011 A A A A A A
6722025-01-27T19:43:19.137Z012 A A A A A A
6732025-01-27T19:43:19.139Z013 A A A A A A
6742025-01-27T19:43:19.140Z014 A A A A A A
6752025-01-27T19:43:19.142Z015 A A A A A A
6762025-01-27T19:43:19.142Z016 A A A A A A
6772025-01-27T19:43:19.145Z017 A A A A A A
6782025-01-27T19:43:19.145Z018 A A A A A A
6792025-01-27T19:43:19.147Z019 A A A A A A
6802025-01-27T19:43:19.150Zdump extent test passed
6812025-01-27T19:43:19.150Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
6822025-01-27T19:43:19.167ZExtent 2 Block in extent 0 Actual block 20
6832025-01-27T19:43:19.170Z
6842025-01-27T19:43:19.170Z DATA SHA256 VER
6852025-01-27T19:43:19.170Z------ ---------------------------------------------------------------- ---
6862025-01-27T19:43:19.173Z 0 825a31d7c30478ff8584e75287eff679da9fc44333843499ea9904c48df1294f A
6872025-01-27T19:43:19.173Z 1 825a31d7c30478ff8584e75287eff679da9fc44333843499ea9904c48df1294f A
6882025-01-27T19:43:19.173Z 2 825a31d7c30478ff8584e75287eff679da9fc44333843499ea9904c48df1294f A
6892025-01-27T19:43:19.173Z
6902025-01-27T19:43:19.173ZNONCES 0 1 2 DIFF
6912025-01-27T19:43:19.173Z------ ------------------------ ------------------------ ------------------------ -----
6922025-01-27T19:43:19.173Z 0 dc6f27287c456d6cd0153de9 dc6f27287c456d6cd0153de9 dc6f27287c456d6cd0153de9
6932025-01-27T19:43:19.173Z
6942025-01-27T19:43:19.173Z TAGS 0 1 2 DIFF
6952025-01-27T19:43:19.173Z------ -------------------------------- -------------------------------- -------------------------------- -----
6962025-01-27T19:43:19.173Z 0 30a8f992b4f0479192285bead9e994c3 30a8f992b4f0479192285bead9e994c3 30a8f992b4f0479192285bead9e994c3
6972025-01-27T19:43:19.173Z
6982025-01-27T19:43:19.173ZHASHES 0 1 2 DIFF
6992025-01-27T19:43:19.173Z------ ---------------- ---------------- ---------------- -----
7002025-01-27T19:43:19.173Z
7012025-01-27T19:43:19.176Zdump block test passed
7022025-01-27T19:43:19.176ZInitial upstairs tests have completed, stopping all downstairs
7032025-01-27T19:43:24.244ZCreating 7 larger downstairs regions
7042025-01-27T19:43:24.434ZStarting 7 downstairs
7052025-01-27T19:43:29.442Zdsc restarted at PID: 1260
7062025-01-27T19:43:29.847ZNow do the replace-reconcile test
7072025-01-27T19:43:29.904ZUsing 8870 for the replacement port
7082025-01-27T19:46:36.342ZCompleted test: replace-reconcile
7092025-01-27T19:46:36.342ZNow do the replace-before-active test
7102025-01-27T19:50:10.023ZCompleted test: replace-before-active
7112025-01-27T19:50:10.023ZAll tests have completed, stopping all downstairs
7122025-01-27T19:50:10.085Z
7132025-01-27T19:50:10.085ZAll Tests have passed
7142025-01-27T19:50:10.088Z13:53 Test duration
7152025-01-27T19:50:10.088Z
7162025-01-27T19:50:10.088Zreal 13:53.067543572
7172025-01-27T19:50:10.088Zuser 18:41.370503346
7182025-01-27T19:50:10.088Zsys 26.432979788
7192025-01-27T19:50:10.089Ztrap 0.562147900
7202025-01-27T19:50:10.089Ztflt 0.041513807
7212025-01-27T19:50:10.089Zdflt 0.189512929
7222025-01-27T19:50:10.089Zkflt 0.000871314
7232025-01-27T19:50:10.089Zlock 25:49:34.180627538
7242025-01-27T19:50:10.089Zslp 1:15:28.241214623
7252025-01-27T19:50:10.089Zlat 2:05.126396341
7262025-01-27T19:50:10.089Zstop 0.206104258
7272025-01-27T19:50:10.089Z+ echo 'test-up-2region-encrypted ends'
7282025-01-27T19:50:10.089Ztest-up-2region-encrypted ends
7292025-01-27T19:50:15.092Zprocess exited: duration 841324 ms, exit code 0
7302025-01-27T19:50:15.093Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
7312025-01-27T19:51:15.142Zexec warning: : stdout descriptor may be held open by a background process; giving up!
7322025-01-27T19:51:15.142Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
7332025-01-27T19:51:15.153Zfound 17 output files
7342025-01-27T19:51:15.153Zuploading: /tmp/test_up-build/dsc-out.txt (52524 bytes)
7352025-01-27T19:51:16.166Zuploaded: /tmp/test_up-build/dsc-out.txt
7362025-01-27T19:51:16.175Zuploading: /tmp/test_up-build/test_up_out.txt (8283051 bytes)
7372025-01-27T19:51:17.474Zuploaded: /tmp/test_up-build/test_up_out.txt
7382025-01-27T19:51:17.477Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (147254 bytes)
7392025-01-27T19:51:18.491Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
7402025-01-27T19:51:18.492Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (144641 bytes)
7412025-01-27T19:51:19.508Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
7422025-01-27T19:51:19.508Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (251047 bytes)
7432025-01-27T19:51:20.528Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
7442025-01-27T19:51:20.531Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (139420 bytes)
7452025-01-27T19:51:21.544Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
7462025-01-27T19:51:21.547Zuploading: /tmp/test_up-build/dsc/downstairs-8850.txt (360784 bytes)
7472025-01-27T19:51:22.567Zuploaded: /tmp/test_up-build/dsc/downstairs-8850.txt
7482025-01-27T19:51:22.570Zuploading: /tmp/test_up-build/dsc/downstairs-8860.txt (36396 bytes)
7492025-01-27T19:51:23.578Zuploaded: /tmp/test_up-build/dsc/downstairs-8860.txt
7502025-01-27T19:51:23.580Zuploading: /tmp/test_up-build/dsc/downstairs-8870.txt (730610 bytes)
7512025-01-27T19:51:24.611Zuploaded: /tmp/test_up-build/dsc/downstairs-8870.txt
7522025-01-27T19:51:24.611Zuploading: /tmp/debug/df.txt (1270 bytes)
7532025-01-27T19:51:25.619Zuploaded: /tmp/debug/df.txt
7542025-01-27T19:51:25.619Zuploading: /tmp/debug/dtrace.txt (990038 bytes)
7552025-01-27T19:51:25.657Zupload warning: file "/tmp/debug/dtrace.txt" changed size mid upload: 990038 -> 1118414
7562025-01-27T19:51:26.662Zuploaded: /tmp/debug/dtrace.txt
7572025-01-27T19:51:26.662Zuploading: /tmp/debug/iostat.txt (269768 bytes)
7582025-01-27T19:51:26.679Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 269768 -> 273068
7592025-01-27T19:51:27.685Zuploaded: /tmp/debug/iostat.txt
7602025-01-27T19:51:27.685Zuploading: /tmp/debug/mpstat.txt (655383 bytes)
7612025-01-27T19:51:27.711Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 655383 -> 664131
7622025-01-27T19:51:28.718Zuploaded: /tmp/debug/mpstat.txt
7632025-01-27T19:51:28.721Zuploading: /tmp/debug/paging.txt (117605 bytes)
7642025-01-27T19:51:28.730Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 117605 -> 119416
7652025-01-27T19:51:29.736Zuploaded: /tmp/debug/paging.txt
7662025-01-27T19:51:29.739Zuploading: /tmp/debug/prstat.txt (1273818 bytes)
7672025-01-27T19:51:29.778Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 1273818 -> 1293225
7682025-01-27T19:51:30.785Zuploaded: /tmp/debug/prstat.txt
7692025-01-27T19:51:30.788Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
7702025-01-27T19:51:31.794Zuploaded: /tmp/debug/psrinfo.txt
7712025-01-27T19:51:31.794Zuploading: /tmp/debug/upstairs-info.txt (186560 bytes)
7722025-01-27T19:51:32.814Zuploaded: /tmp/debug/upstairs-info.txt