01JJA2YXHK1K091HM7SWVY1PDN: test-up-2region-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01JJA2Z8WSCKDJ0HFMGQN22NTK

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12025-01-23T17:36:06.963Zjob dependencies complete; ready to run (waiting for 13 m 49 s)
22025-01-23T17:37:15.220Zjob assigned to worker 01JJA3RXZDDZM3TEBFC9AVDATJ [factory aws, i-0bf5e820e1cb25f18] (queued for 1 m 8 s)
32025-01-23T17:37:23.920Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42025-01-23T17:37:25.086Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52025-01-23T17:37:25.087Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62025-01-23T17:37:26.192Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72025-01-23T17:37:26.192Zdownloading input: /input/build/work/bins/crutest.gz
82025-01-23T17:37:27.328Zdownloaded input: /input/build/work/bins/crutest.gz
92025-01-23T17:37:27.328Zdownloading input: /input/build/work/bins/dsc.gz
102025-01-23T17:37:28.030Zdownloaded input: /input/build/work/bins/dsc.gz
112025-01-23T17:37:28.033Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122025-01-23T17:37:28.049Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132025-01-23T17:37:28.049Zdownloading input: /input/build/work/scripts/test_ds.sh
142025-01-23T17:37:28.089Zdownloaded input: /input/build/work/scripts/test_ds.sh
152025-01-23T17:37:28.089Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162025-01-23T17:37:28.105Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172025-01-23T17:37:28.105Zdownloading input: /input/build/work/scripts/test_repair.sh
182025-01-23T17:37:28.124Zdownloaded input: /input/build/work/scripts/test_repair.sh
192025-01-23T17:37:28.124Zdownloading input: /input/build/work/scripts/test_replay.sh
202025-01-23T17:37:28.141Zdownloaded input: /input/build/work/scripts/test_replay.sh
212025-01-23T17:37:28.141Zdownloading input: /input/build/work/scripts/test_up.sh
222025-01-23T17:37:28.162Zdownloaded input: /input/build/work/scripts/test_up.sh
232025-01-23T17:37:28.162Zdownloading input: /input/build/work/scripts/upstairs_info.d
242025-01-23T17:37:28.181Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252025-01-23T17:37:28.184Zdownloading input: /input/build/tmp/cargo-test-out.log
262025-01-23T17:37:28.891Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272025-01-23T17:37:28.891Zstarting task 0: "setup"
282025-01-23T17:37:28.895Z++ uname -s
292025-01-23T17:37:28.895Z+ kern=SunOS
302025-01-23T17:37:28.895Z+ case "$kern" in
312025-01-23T17:37:28.895Z+ groupadd -g 12345 build
322025-01-23T17:37:28.901Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332025-01-23T17:37:30.906Z+ zfs create -o mountpoint=/work rpool/work
342025-01-23T17:37:31.047Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352025-01-23T17:37:31.051Z+ home_fs=zfs
362025-01-23T17:37:31.051Z+ [[ zfs == autofs ]]
372025-01-23T17:37:31.051Z+ mkdir -p /home/build
382025-01-23T17:37:31.054Z+ chown build:build /home/build /work
392025-01-23T17:37:33.054Z+ chmod 0700 /home/build /work
402025-01-23T17:37:33.057Zprocess exited: duration 4174 ms, exit code 0
 
412025-01-23T17:37:33.062Zstarting task 1: "authentication"
422025-01-23T17:37:33.080Zprocess exited: duration 17 ms, exit code 0
 
432025-01-23T17:37:33.088Zstarting task 2: "build"
442025-01-23T17:37:33.091Z+ banner cores
452025-01-23T17:37:33.107Z
462025-01-23T17:37:33.107Z #### #### ##### ###### ####
472025-01-23T17:37:33.107Z # # # # # # # #
482025-01-23T17:37:33.107Z # # # # # ##### ####
492025-01-23T17:37:33.107Z # # # ##### # #
502025-01-23T17:37:33.107Z # # # # # # # # #
512025-01-23T17:37:33.107Z #### #### # # ###### ####
522025-01-23T17:37:33.107Z
532025-01-23T17:37:33.110Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542025-01-23T17:37:33.116Z+ banner unpack
552025-01-23T17:37:34.121Z
562025-01-23T17:37:34.121Z # # # # ##### ## #### # #
572025-01-23T17:37:34.121Z # # ## # # # # # # # # #
582025-01-23T17:37:34.121Z # # # # # # # # # # ####
592025-01-23T17:37:34.121Z # # # # # ##### ###### # # #
602025-01-23T17:37:34.121Z # # # ## # # # # # # #
612025-01-23T17:37:34.121Z #### # # # # # #### # #
622025-01-23T17:37:34.121Z
632025-01-23T17:37:34.121Z+ mkdir -p /var/tmp/bins
642025-01-23T17:37:34.125Z+ for t in "$input/bins/"*.gz
652025-01-23T17:37:34.125Z++ basename /input/build/work/bins/crucible-downstairs.gz
662025-01-23T17:37:34.125Z+ b=crucible-downstairs.gz
672025-01-23T17:37:34.125Z+ b=crucible-downstairs
682025-01-23T17:37:34.125Z+ gunzip
692025-01-23T17:37:35.467Z+ chmod +x /var/tmp/bins/crucible-downstairs
702025-01-23T17:37:35.470Z+ for t in "$input/bins/"*.gz
712025-01-23T17:37:35.473Z++ basename /input/build/work/bins/crucible-hammer.gz
722025-01-23T17:37:35.473Z+ b=crucible-hammer.gz
732025-01-23T17:37:35.473Z+ b=crucible-hammer
742025-01-23T17:37:35.474Z+ gunzip
752025-01-23T17:37:37.605Z+ chmod +x /var/tmp/bins/crucible-hammer
762025-01-23T17:37:37.608Z+ for t in "$input/bins/"*.gz
772025-01-23T17:37:37.610Z++ basename /input/build/work/bins/crutest.gz
782025-01-23T17:37:37.610Z+ b=crutest.gz
792025-01-23T17:37:37.610Z+ b=crutest
802025-01-23T17:37:37.610Z+ gunzip
812025-01-23T17:37:40.020Z+ chmod +x /var/tmp/bins/crutest
822025-01-23T17:37:40.024Z+ for t in "$input/bins/"*.gz
832025-01-23T17:37:40.026Z++ basename /input/build/work/bins/dsc.gz
842025-01-23T17:37:40.026Z+ b=dsc.gz
852025-01-23T17:37:40.026Z+ b=dsc
862025-01-23T17:37:40.026Z+ gunzip
872025-01-23T17:37:41.375Z+ chmod +x /var/tmp/bins/dsc
882025-01-23T17:37:41.378Z+ export BINDIR=/var/tmp/bins
892025-01-23T17:37:41.378Z+ BINDIR=/var/tmp/bins
902025-01-23T17:37:41.378Z+ jobpid=1141
912025-01-23T17:37:41.378Z+ echo 'Setup debug logging'
922025-01-23T17:37:41.379ZSetup debug logging
932025-01-23T17:37:41.379Z+ mkdir /tmp/debug
942025-01-23T17:37:41.379Z+ sleep 3600
952025-01-23T17:37:41.382Z+ psrinfo -v
962025-01-23T17:37:41.385Z+ df -h
972025-01-23T17:37:41.404Z+ prstat -d d -mLc 1
982025-01-23T17:37:41.407Z+ iostat -T d -xn 1
992025-01-23T17:37:41.407Z+ mpstat -T d 1
1002025-01-23T17:37:41.407Z+ vmstat -T d -p 1
1012025-01-23T17:37:41.410Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1022025-01-23T17:37:41.410Z+ banner test_up_2r_encrypted
1032025-01-23T17:37:41.410Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1042025-01-23T17:37:41.410Z #####
1052025-01-23T17:37:41.410Z ##### ###### #### ##### # # ##### # # #####
1062025-01-23T17:37:41.410Z # # # # # # # # # # #
1072025-01-23T17:37:41.410Z # ##### #### # # # # # ##### # #
1082025-01-23T17:37:41.410Z # # # # # # ##### # #####
1092025-01-23T17:37:41.410Z # # # # # # # # # # #
1102025-01-23T17:37:41.410Z # ###### #### # ####### #### # ####### ####### # #
1112025-01-23T17:37:41.410Z
1122025-01-23T17:37:41.410Z+ ptime -m bash /input/build/work/scripts/test_up.sh -r 2 -N encrypted
1132025-01-23T17:37:41.421Z/input/build/work
1142025-01-23T17:37:41.445ZUsing 2 region sets
1152025-01-23T17:37:41.445ZTurn off color for downstairs dump
1162025-01-23T17:37:41.461ZUpstairs using key: ZkhwTgnB1LwVexb0z+8gydsSKERmT9kymvTEBkA7fYI=
1172025-01-23T17:37:41.464Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1182025-01-23T17:37:41.464ZCreating 6 downstairs regions
1192025-01-23T17:37:41.830ZStarting 6 downstairs
1202025-01-23T17:37:41.833Zdsc started at PID: 1189
1212025-01-23T17:37:47.192ZDisable automatic restart on all downstairs
1222025-01-23T17:37:47.248Z
1232025-01-23T17:37:47.248ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1242025-01-23T17:37:47.252ZRunning test: span
1252025-01-23T17:37:51.428ZCompleted test: span
1262025-01-23T17:37:56.436ZRunning test: big
1272025-01-23T17:38:01.009ZCompleted test: big
1282025-01-23T17:38:06.017ZRunning test: dep
1292025-01-23T17:38:39.693ZCompleted test: dep
1302025-01-23T17:38:44.702ZRunning test: balloon
1312025-01-23T17:42:55.910ZCompleted test: balloon
1322025-01-23T17:43:00.918ZRunning test: deactivate
1332025-01-23T17:43:55.684ZCompleted test: deactivate
1342025-01-23T17:44:00.692ZRunning hammer
1352025-01-23T17:44:13.403ZRun repair tests
1362025-01-23T17:44:13.407Z/var/tmp/bins/crutest fill -g 31 -q --key ZkhwTgnB1LwVexb0z+8gydsSKERmT9kymvTEBkA7fYI= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1372025-01-23T17:44:13.467Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.161197807Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"task":"crutest"}
1382025-01-23T17:44:13.470Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.161780993Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"task":"crutest"}
1392025-01-23T17:44:13.470Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.163442547Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"task":"crutest"}
1402025-01-23T17:44:13.470Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.163657785Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232}
1412025-01-23T17:44:13.470Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.163703296Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232}
1422025-01-23T17:44:13.470Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.163727022Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232}
1432025-01-23T17:44:13.470Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.163742159Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232}
1442025-01-23T17:44:13.470Z{"msg":"Crucible c4d30283-8c39-44b3-91af-30dea6f11a8e has session id: 8551c251-0b0c-461b-adb8-dc96c82247c8","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.164480984Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1452025-01-23T17:44:13.470Z{"msg":"Upstairs opts: Upstairs UUID: c4d30283-8c39-44b3-91af-30dea6f11a8e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.164545491Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1462025-01-23T17:44:13.470Z{"msg":"Crucible stats registered with UUID: c4d30283-8c39-44b3-91af-30dea6f11a8e","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.164572746Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1472025-01-23T17:44:13.470Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:12.164620826Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1482025-01-23T17:44:13.473Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.166647979Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"task":"crutest"}
1492025-01-23T17:44:13.473Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.166731872Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232}
1502025-01-23T17:44:13.473Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.166769015Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232}
1512025-01-23T17:44:13.473Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.166789191Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232}
1522025-01-23T17:44:13.473Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.166804718Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232}
1532025-01-23T17:44:13.473Z{"msg":"Crucible c4d30283-8c39-44b3-91af-30dea6f11a8e has session id: c470522b-8737-4b5f-a02f-a4e9b0ad5f07","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.167239863Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
1542025-01-23T17:44:13.473Z{"msg":"Upstairs opts: Upstairs UUID: c4d30283-8c39-44b3-91af-30dea6f11a8e, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.167288883Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
1552025-01-23T17:44:13.474Z{"msg":"Crucible stats registered with UUID: c4d30283-8c39-44b3-91af-30dea6f11a8e","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.167316668Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
1562025-01-23T17:44:13.474Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:12.167410719Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
1572025-01-23T17:44:13.477Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.168058783Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"task":"crutest"}
1582025-01-23T17:44:13.478Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.168179719Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232}
1592025-01-23T17:44:13.478Z{"msg":"c4d30283-8c39-44b3-91af-30dea6f11a8e active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.16822406Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1602025-01-23T17:44:13.478Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.168287168Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"client":"0","":"downstairs","session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1612025-01-23T17:44:13.478Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.168347796Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"client":"0","":"downstairs","session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1622025-01-23T17:44:13.478Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.168413983Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"client":"1","":"downstairs","session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1632025-01-23T17:44:13.478Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.168448716Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"client":"1","":"downstairs","session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1642025-01-23T17:44:13.478Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.168476311Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"client":"2","":"downstairs","session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1652025-01-23T17:44:13.478Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.168504725Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"client":"2","":"downstairs","session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1662025-01-23T17:44:13.478Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.1685278Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1672025-01-23T17:44:13.478Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.168568123Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1682025-01-23T17:44:13.478Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.168686349Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1692025-01-23T17:44:13.478Z{"msg":"ds_connection connected from Ok(127.0.0.1:54218)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.168764884Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1702025-01-23T17:44:13.478Z{"msg":"ds_connection connected from Ok(127.0.0.1:49629)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.168793099Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1712025-01-23T17:44:13.479Z{"msg":"ds_connection connected from Ok(127.0.0.1:60378)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.16883564Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1722025-01-23T17:44:13.479Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.169212616Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"client":"0","":"downstairs","session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1732025-01-23T17:44:13.479Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.169260857Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"client":"1","":"downstairs","session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1742025-01-23T17:44:13.479Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.169300419Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"client":"2","":"downstairs","session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1752025-01-23T17:44:13.479Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.169839314Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"client":"1","":"downstairs","session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1762025-01-23T17:44:13.479Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.169999132Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"client":"2","":"downstairs","session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1772025-01-23T17:44:13.479Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.170072048Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"client":"0","":"downstairs","session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1782025-01-23T17:44:13.479Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.17021539Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1792025-01-23T17:44:13.479Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.170269029Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1802025-01-23T17:44:13.479Z{"msg":"[0]R flush_numbers: [3994, 3994, 3994, 3994, 3994]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.170396454Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"downstairs","":"downstairs","session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1812025-01-23T17:44:13.479Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.170424709Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"downstairs","":"downstairs","session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1822025-01-23T17:44:13.479Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.170440396Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"downstairs","":"downstairs","session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1832025-01-23T17:44:13.479Z{"msg":"[1]R flush_numbers: [3994, 3994, 3994, 3994, 3994]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.170461601Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"downstairs","":"downstairs","session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1842025-01-23T17:44:13.479Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.170474949Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"downstairs","":"downstairs","session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1852025-01-23T17:44:13.479Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.170495985Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"downstairs","":"downstairs","session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1862025-01-23T17:44:13.479Z{"msg":"[2]R flush_numbers: [3994, 3994, 3994, 3994, 3994]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.170509512Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"downstairs","":"downstairs","session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1872025-01-23T17:44:13.479Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.170530158Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"downstairs","":"downstairs","session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1882025-01-23T17:44:13.479Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.170543525Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"downstairs","":"downstairs","session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1892025-01-23T17:44:13.479Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.170563252Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"downstairs","":"downstairs","session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1902025-01-23T17:44:13.479Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.170576369Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"downstairs","":"downstairs","session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1912025-01-23T17:44:13.479Z{"msg":"Next flush: 3995","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.170596025Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"downstairs","":"downstairs","session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1922025-01-23T17:44:13.479Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.170616061Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"downstairs","":"downstairs","session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1932025-01-23T17:44:13.479Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.170635387Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1942025-01-23T17:44:13.479Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.170668011Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"client":"0","":"downstairs","session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1952025-01-23T17:44:13.479Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.170705984Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"client":"1","":"downstairs","session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1962025-01-23T17:44:13.479Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.170747125Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"client":"2","":"downstairs","session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1972025-01-23T17:44:13.479Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.170778209Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1982025-01-23T17:44:13.479Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.170813942Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
1992025-01-23T17:44:13.479Z{"msg":"c4d30283-8c39-44b3-91af-30dea6f11a8e is now active with session: 8551c251-0b0c-461b-adb8-dc96c82247c8","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.170867342Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
2002025-01-23T17:44:13.479Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.170954585Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"session_id":"8551c251-0b0c-461b-adb8-dc96c82247c8"}
2012025-01-23T17:44:13.479Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.17097746Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232}
2022025-01-23T17:44:13.479Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.170998226Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232}
2032025-01-23T17:44:13.479Z{"msg":"c4d30283-8c39-44b3-91af-30dea6f11a8e active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.171020462Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2042025-01-23T17:44:13.479Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.171044937Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"client":"0","":"downstairs","session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2052025-01-23T17:44:13.479Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.17108397Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"client":"0","":"downstairs","session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2062025-01-23T17:44:13.479Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.171099476Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"client":"1","":"downstairs","session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2072025-01-23T17:44:13.479Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.171125661Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"client":"1","":"downstairs","session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2082025-01-23T17:44:13.480Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.171141808Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"client":"2","":"downstairs","session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2092025-01-23T17:44:13.480Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.171188979Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"client":"2","":"downstairs","session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2102025-01-23T17:44:13.480Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.171208445Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2112025-01-23T17:44:13.480Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.171226862Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2122025-01-23T17:44:13.483Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.171252097Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2132025-01-23T17:44:13.483Z{"msg":"ds_connection connected from Ok(127.0.0.1:56543)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.171288559Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2142025-01-23T17:44:13.483Z{"msg":"ds_connection connected from Ok(127.0.0.1:41461)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.171314694Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2152025-01-23T17:44:13.483Z{"msg":"ds_connection connected from Ok(127.0.0.1:47479)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.171350397Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2162025-01-23T17:44:13.483Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.17138647Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"client":"0","":"downstairs","session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2172025-01-23T17:44:13.483Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.171402357Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"client":"2","":"downstairs","session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2182025-01-23T17:44:13.483Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.171420014Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"client":"1","":"downstairs","session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2192025-01-23T17:44:13.483Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.171711496Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"client":"0","":"downstairs","session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2202025-01-23T17:44:13.484Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.171808548Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"client":"2","":"downstairs","session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2212025-01-23T17:44:13.484Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.171850389Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"client":"1","":"downstairs","session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2222025-01-23T17:44:13.484Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.172017747Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2232025-01-23T17:44:13.484Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.17215296Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2242025-01-23T17:44:13.484Z{"msg":"[0]R flush_numbers: [5315, 5315, 5316, 5316, 5311]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.172238283Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"downstairs","":"downstairs","session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2252025-01-23T17:44:13.484Z{"msg":"[0]R generation: [14, 14, 14, 14, 12]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.172264358Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"downstairs","":"downstairs","session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2262025-01-23T17:44:13.484Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.172291753Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"downstairs","":"downstairs","session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2272025-01-23T17:44:13.484Z{"msg":"[1]R flush_numbers: [5315, 5315, 5316, 5316, 5311]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.172314148Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"downstairs","":"downstairs","session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2282025-01-23T17:44:13.484Z{"msg":"[1]R generation: [14, 14, 14, 14, 12]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.172328226Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"downstairs","":"downstairs","session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2292025-01-23T17:44:13.484Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.172350321Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"downstairs","":"downstairs","session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2302025-01-23T17:44:13.484Z{"msg":"[2]R flush_numbers: [5315, 5315, 5316, 5316, 5311]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.172364639Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"downstairs","":"downstairs","session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2312025-01-23T17:44:13.484Z{"msg":"[2]R generation: [14, 14, 14, 14, 12]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.172377266Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"downstairs","":"downstairs","session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2322025-01-23T17:44:13.484Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
2332025-01-23T17:44:13.484Z,"time":"2025-01-23T17:44:12.172397582Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"downstairs","":"downstairs","session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2342025-01-23T17:44:13.484Z{"msg":"Max found gen is 15","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.172413329Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"downstairs","":"downstairs","session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2352025-01-23T17:44:13.484Z{"msg":"Generation requested: 31 >= found:15","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.172434625Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"downstairs","":"downstairs","session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2362025-01-23T17:44:13.484Z{"msg":"Next flush: 5317","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.172448012Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"downstairs","":"downstairs","session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2372025-01-23T17:44:13.484Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.172473297Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"":"downstairs","":"downstairs","session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2382025-01-23T17:44:13.484Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.172486695Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2392025-01-23T17:44:13.484Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.172507291Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"client":"0","":"downstairs","session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2402025-01-23T17:44:13.484Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.172521668Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"client":"1","":"downstairs","session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2412025-01-23T17:44:13.484Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.172534915Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"client":"2","":"downstairs","session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2422025-01-23T17:44:13.484Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.172547643Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2432025-01-23T17:44:13.484Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.172567219Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2442025-01-23T17:44:13.484Z{"msg":"c4d30283-8c39-44b3-91af-30dea6f11a8e is now active with session: c470522b-8737-4b5f-a02f-a4e9b0ad5f07","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.172580226Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2452025-01-23T17:44:13.484Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.172592544Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232,"session_id":"c470522b-8737-4b5f-a02f-a4e9b0ad5f07"}
2462025-01-23T17:44:13.484Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:12.172605181Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232}
2472025-01-23T17:44:13.484ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
2482025-01-23T17:44:13.484ZFill test
2492025-01-23T17:44:13.520ZRead and Verify all blocks (0..100 range:false)
2502025-01-23T17:44:13.601ZWrote out file "/var/tmp/test_up-build/verify_file"
2512025-01-23T17:44:13.601ZCLIENT: Tests done. All submitted work has been ACK'd
2522025-01-23T17:44:13.604Z----------------------------------------------------------------
2532025-01-23T17:44:13.604Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2542025-01-23T17:44:13.604ZStates: Active Active Active
2552025-01-23T17:44:13.604ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2562025-01-23T17:44:13.604Z 1002 Acked Read 50 Done Done Done false
2572025-01-23T17:44:13.605Z STATES DS:0 DS:1 DS:2 TOTAL
2582025-01-23T17:44:13.605Z Sent 0 0 0 0
2592025-01-23T17:44:13.605Z Done 1 1 1 3
2602025-01-23T17:44:13.605Z Skipped 0 0 0 0
2612025-01-23T17:44:13.605Z Error 0 0 0 0
2622025-01-23T17:44:13.605ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2632025-01-23T17:44:13.605ZDownstairs last five completed: 1001 1000
2642025-01-23T17:44:13.605ZUpstairs last five completed: 1002 1001 1000
2652025-01-23T17:44:13.605Z----------------------------------------------------------------
2662025-01-23T17:44:13.605Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2672025-01-23T17:44:13.605ZStates: Active Active Active
2682025-01-23T17:44:13.605ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2692025-01-23T17:44:13.605Z 1002 Acked Read 50 Done Done Done false
2702025-01-23T17:44:13.605Z STATES DS:0 DS:1 DS:2 TOTAL
2712025-01-23T17:44:13.605Z Sent 0 0 0 0
2722025-01-23T17:44:13.605Z Done 1 1 1 3
2732025-01-23T17:44:13.605Z Skipped 0 0 0 0
2742025-01-23T17:44:13.605Z Error 0 0 0 0
2752025-01-23T17:44:13.605ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2762025-01-23T17:44:13.605ZDownstairs last five completed: 1001 1000
2772025-01-23T17:44:13.605ZUpstairs last five completed: 1002 1001 1000
2782025-01-23T17:44:13.605ZCLIENT: Up:0 ds:2 act:6
2792025-01-23T17:44:17.604Z----------------------------------------------------------------
2802025-01-23T17:44:17.604Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2812025-01-23T17:44:17.604ZDownstairs last five completed: 1003 1002 1001 1000
2822025-01-23T17:44:17.604ZUpstairs last five completed: 1003 1002 1001 1000
2832025-01-23T17:44:17.604Z----------------------------------------------------------------
2842025-01-23T17:44:17.604Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2852025-01-23T17:44:17.604ZDownstairs last five completed: 1003 1002 1001 1000
2862025-01-23T17:44:17.604ZUpstairs last five completed: 1003 1002 1001 1000
2872025-01-23T17:44:17.607ZCLIENT: Up:0 ds:0 act:6
2882025-01-23T17:44:17.607ZCLIENT: All crucible jobs finished, exiting program
2892025-01-23T17:44:17.607Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:16.297603086Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1232}
2902025-01-23T17:44:17.615ZRepair setup passed
2912025-01-23T17:44:17.619ZCopy the region for /var/tmp/test_up-build/8830
2922025-01-23T17:44:17.621Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2932025-01-23T17:44:17.624Z/var/tmp/bins/crutest fill -g 32 -q --key ZkhwTgnB1LwVexb0z+8gydsSKERmT9kymvTEBkA7fYI= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2942025-01-23T17:44:17.682Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.375770054Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"task":"crutest"}
2952025-01-23T17:44:17.686Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.376396461Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"task":"crutest"}
2962025-01-23T17:44:17.686Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.378037799Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"task":"crutest"}
2972025-01-23T17:44:17.686Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.378226972Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238}
2982025-01-23T17:44:17.686Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.378272653Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238}
2992025-01-23T17:44:17.686Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.378296108Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238}
3002025-01-23T17:44:17.686Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.378310805Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238}
3012025-01-23T17:44:17.686Z{"msg":"Crucible e3db09c9-85e2-43d3-8d2b-b999381e1892 has session id: 24dae51a-6b18-4a79-8c19-4132d179fd4c","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.37885032Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3022025-01-23T17:44:17.686Z{"msg":"Upstairs opts: Upstairs UUID: e3db09c9-85e2-43d3-8d2b-b999381e1892, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.378915667Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3032025-01-23T17:44:17.686Z{"msg":"Crucible stats registered with UUID: e3db09c9-85e2-43d3-8d2b-b999381e1892","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.378941262Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3042025-01-23T17:44:17.686Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:16.378969307Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3052025-01-23T17:44:17.689Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.381173584Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"task":"crutest"}
3062025-01-23T17:44:17.689Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.381283633Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238}
3072025-01-23T17:44:17.689Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.381327434Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238}
3082025-01-23T17:44:17.689Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.38134951Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238}
3092025-01-23T17:44:17.689Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.381364837Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238}
3102025-01-23T17:44:17.689Z{"msg":"Crucible e3db09c9-85e2-43d3-8d2b-b999381e1892 has session id: dae722a3-a077-4544-a6c2-30f34313cd56","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.381744203Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3112025-01-23T17:44:17.689Z{"msg":"Upstairs opts: Upstairs UUID: e3db09c9-85e2-43d3-8d2b-b999381e1892, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.38180594Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3122025-01-23T17:44:17.689Z{"msg":"Crucible stats registered with UUID: e3db09c9-85e2-43d3-8d2b-b999381e1892","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.381840934Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3132025-01-23T17:44:17.689Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:16.381862429Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3142025-01-23T17:44:17.690Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.382555164Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"task":"crutest"}
3152025-01-23T17:44:17.690Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.382593096Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238}
3162025-01-23T17:44:17.690Z{"msg":"e3db09c9-85e2-43d3-8d2b-b999381e1892 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.382683248Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3172025-01-23T17:44:17.690Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.382732789Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"client":"0","":"downstairs","session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3182025-01-23T17:44:17.690Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.382797216Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"client":"0","":"downstairs","session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3192025-01-23T17:44:17.694Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.382842267Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"client":"1","":"downstairs","session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3202025-01-23T17:44:17.694Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.38288266Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"client":"1","":"downstairs","session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3212025-01-23T17:44:17.694Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.38298085Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"client":"2","":"downstairs","session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3222025-01-23T17:44:17.694Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.383016233Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"client":"2","":"downstairs","session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3232025-01-23T17:44:17.694Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.383038849Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3242025-01-23T17:44:17.694Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.383189479Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3252025-01-23T17:44:17.694Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.383245338Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3262025-01-23T17:44:17.694Z{"msg":"ds_connection connected from Ok(127.0.0.1:38607)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.383271243Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3272025-01-23T17:44:17.695Z{"msg":"ds_connection connected from Ok(127.0.0.1:36545)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.383305617Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3282025-01-23T17:44:17.695Z{"msg":"ds_connection connected from Ok(127.0.0.1:40325)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.383350988Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3292025-01-23T17:44:17.695Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.38379691Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"client":"1","":"downstairs","session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3302025-01-23T17:44:17.695Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.383875135Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"client":"2","":"downstairs","session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3312025-01-23T17:44:17.695Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.383917586Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"client":"0","":"downstairs","session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3322025-01-23T17:44:17.695Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.384328466Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"client":"1","":"downstairs","session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3332025-01-23T17:44:17.696Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.384442674Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"client":"0","":"downstairs","session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3342025-01-23T17:44:17.696Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.384530556Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"client":"2","":"downstairs","session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3352025-01-23T17:44:17.696Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.38471474Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3362025-01-23T17:44:17.696Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.384796794Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3372025-01-23T17:44:17.696Z{"msg":"[0]R flush_numbers: [3995, 3995, 3995, 3995, 3995]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.384908942Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"downstairs","":"downstairs","session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3382025-01-23T17:44:17.696Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.384936907Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"downstairs","":"downstairs","session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3392025-01-23T17:44:17.696Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.384952614Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"downstairs","":"downstairs","session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3402025-01-23T17:44:17.696Z{"msg":"[1]R flush_numbers: [3995, 3995, 3995, 3995, 3995]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.384966051Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"downstairs","":"downstairs","session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3412025-01-23T17:44:17.696Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.384985167Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"downstairs","":"downstairs","session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3422025-01-23T17:44:17.696Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.385009332Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"downstairs","":"downstairs","session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3432025-01-23T17:44:17.696Z{"msg":"[2]R flush_numbers: [3995, 3995, 3995, 3995, 3995]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.385044106Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"downstairs","":"downstairs","session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3442025-01-23T17:44:17.696Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.385070121Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"downstairs","":"downstairs","session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3452025-01-23T17:44:17.696Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.385108753Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"downstairs","":"downstairs","session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3462025-01-23T17:44:17.696Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.385135948Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"downstairs","":"downstairs","session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3472025-01-23T17:44:17.696Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.385172351Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"downstairs","":"downstairs","session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3482025-01-23T17:44:17.696Z{"msg":"Next flush: 3996","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.385195986Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"downstairs","":"downstairs","session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3492025-01-23T17:44:17.696Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.385219022Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"downstairs","":"downstairs","session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3502025-01-23T17:44:17.696Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.385233239Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3512025-01-23T17:44:17.696Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.385264033Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"client":"0","":"downstairs","session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3522025-01-23T17:44:17.696Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.385288148Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"client":"1","":"downstairs","session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3532025-01-23T17:44:17.696Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.385320891Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"client":"2","":"downstairs","session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3542025-01-23T17:44:17.696Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.385351045Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3552025-01-23T17:44:17.696Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.385375681Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3562025-01-23T17:44:17.696Z{"msg":"e3db09c9-85e2-43d3-8d2b-b999381e1892 is now active with session: 24dae51a-6b18-4a79-8c19-4132d179fd4c","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.385400986Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3572025-01-23T17:44:17.696Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.385554116Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"session_id":"24dae51a-6b18-4a79-8c19-4132d179fd4c"}
3582025-01-23T17:44:17.697Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.385592238Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238}
3592025-01-23T17:44:17.697Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.385606695Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238}
3602025-01-23T17:44:17.697Z{"msg":"e3db09c9-85e2-43d3-8d2b-b999381e1892 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.385618093Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3612025-01-23T17:44:17.697Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.38563645Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"client":"0","":"downstairs","session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3622025-01-23T17:44:17.697Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.385664924Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"client":"0","":"downstairs","session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3632025-01-23T17:44:17.697Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.385679771Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"client":"1","":"downstairs","session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3642025-01-23T17:44:17.697Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.385694338Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"client":"1","":"downstairs","session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3652025-01-23T17:44:17.697Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.385717824Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"client":"2","":"downstairs","session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3662025-01-23T17:44:17.697Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.385744359Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"client":"2","":"downstairs","session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3672025-01-23T17:44:17.697Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.385782751Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3682025-01-23T17:44:17.697Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.385819904Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3692025-01-23T17:44:17.697Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.385847068Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3702025-01-23T17:44:17.697Z{"msg":"ds_connection connected from Ok(127.0.0.1:35118)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.385863235Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3712025-01-23T17:44:17.697Z{"msg":"ds_connection connected from Ok(127.0.0.1:35948)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.385885521Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3722025-01-23T17:44:17.697Z{"msg":"ds_connection connected from Ok(127.0.0.1:52020)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.385899728Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3732025-01-23T17:44:17.697Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.385913845Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"client":"0","":"downstairs","session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3742025-01-23T17:44:17.697Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.385954727Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"client":"1","":"downstairs","session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3752025-01-23T17:44:17.697Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.3860427Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"client":"2","":"downstairs","session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3762025-01-23T17:44:17.697Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.386318846Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"client":"0","":"downstairs","session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3772025-01-23T17:44:17.701Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.386479084Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"client":"1","":"downstairs","session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3782025-01-23T17:44:17.701Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.386583814Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"client":"2","":"downstairs","session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3792025-01-23T17:44:17.701Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.386782185Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3802025-01-23T17:44:17.701Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.38691061Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3812025-01-23T17:44:17.701Z{"msg":"[0]R flush_numbers: [5317, 5317, 5317, 5317, 5317]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.386957151Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"downstairs","":"downstairs","session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3822025-01-23T17:44:17.701Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.387003832Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"downstairs","":"downstairs","session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3832025-01-23T17:44:17.701Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.387019329Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"downstairs","":"downstairs","session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3842025-01-23T17:44:17.701Z{"msg":"[1]R flush_numbers: [5317, 5317, 5317, 5317, 5317]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.387040114Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"downstairs","":"downstairs","session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3852025-01-23T17:44:17.701Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.387053872Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"downstairs","":"downstairs",Wait for a query_work_queue command to finish before sending IO
3862025-01-23T17:44:17.701Z"session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3872025-01-23T17:44:17.701Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.387081246Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"downstairs","":"downstairs","session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3882025-01-23T17:44:17.701Z{"msg":"[2]R flush_numbers: [5317, 5317, 5317, 5317, 5317]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.387095004Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"downstairs","":"downstairs","session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3892025-01-23T17:44:17.701Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.387115999Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"downstairs","":"downstairs","session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3902025-01-23T17:44:17.701Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.387129627Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"downstairs","":"downstairs","session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3912025-01-23T17:44:17.701Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.387149823Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"downstairs","":"downstairs","session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3922025-01-23T17:44:17.701Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.38716567Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"downstairs","":"downstairs","session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3932025-01-23T17:44:17.701Z{"msg":"Next flush: 5318","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.387191645Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"downstairs","":"downstairs","session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3942025-01-23T17:44:17.701Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.387205482Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"":"downstairs","":"downstairs","session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3952025-01-23T17:44:17.701Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.387225868Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3962025-01-23T17:44:17.701Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.387238116Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"client":"0","":"downstairs","session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3972025-01-23T17:44:17.701Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.38726789Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"client":"1","":"downstairs","session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3982025-01-23T17:44:17.701Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.387315151Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"client":"2","":"downstairs","session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
3992025-01-23T17:44:17.701Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.387379188Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
4002025-01-23T17:44:17.701Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.387400194Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
4012025-01-23T17:44:17.701Z{"msg":"e3db09c9-85e2-43d3-8d2b-b999381e1892 is now active with session: dae722a3-a077-4544-a6c2-30f34313cd56","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.38746961Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
4022025-01-23T17:44:17.701Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.387495575Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238,"session_id":"dae722a3-a077-4544-a6c2-30f34313cd56"}
4032025-01-23T17:44:17.701Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:16.387518581Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238}
4042025-01-23T17:44:17.701ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
4052025-01-23T17:44:17.702ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4062025-01-23T17:44:17.702ZFill test
4072025-01-23T17:44:17.736ZRead and Verify all blocks (0..100 range:false)
4082025-01-23T17:44:17.828ZWrote out file "/var/tmp/test_up-build/verify_file"
4092025-01-23T17:44:17.828ZCLIENT: Tests done. All submitted work has been ACK'd
4102025-01-23T17:44:17.828Z----------------------------------------------------------------
4112025-01-23T17:44:17.828Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4122025-01-23T17:44:17.828ZStates: Active Active Active
4132025-01-23T17:44:17.831ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4142025-01-23T17:44:17.831Z 1002 Acked Read 50 Done Done Done false
4152025-01-23T17:44:17.831Z STATES DS:0 DS:1 DS:2 TOTAL
4162025-01-23T17:44:17.831Z Sent 0 0 0 0
4172025-01-23T17:44:17.831Z Done 1 1 1 3
4182025-01-23T17:44:17.831Z Skipped 0 0 0 0
4192025-01-23T17:44:17.831Z Error 0 0 0 0
4202025-01-23T17:44:17.831ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4212025-01-23T17:44:17.832ZDownstairs last five completed: 1001 1000
4222025-01-23T17:44:17.832ZUpstairs last five completed: 1002 1001 1000
4232025-01-23T17:44:17.832Z----------------------------------------------------------------
4242025-01-23T17:44:17.832Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4252025-01-23T17:44:17.832ZStates: Active Active Active
4262025-01-23T17:44:17.832ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4272025-01-23T17:44:17.832Z 1002 Acked Read 50 Done Done Done false
4282025-01-23T17:44:17.832Z STATES DS:0 DS:1 DS:2 TOTAL
4292025-01-23T17:44:17.832Z Sent 0 0 0 0
4302025-01-23T17:44:17.832Z Done 1 1 1 3
4312025-01-23T17:44:17.832Z Skipped 0 0 0 0
4322025-01-23T17:44:17.832Z Error 0 0 0 0
4332025-01-23T17:44:17.832ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4342025-01-23T17:44:17.832ZDownstairs last five completed: 1001 1000
4352025-01-23T17:44:17.832ZUpstairs last five completed: 1002 1001 1000
4362025-01-23T17:44:17.832ZCLIENT: Up:0 ds:2 act:6
4372025-01-23T17:44:21.832Z----------------------------------------------------------------
4382025-01-23T17:44:21.832Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4392025-01-23T17:44:21.832ZDownstairs last five completed: 1003 1002 1001 1000
4402025-01-23T17:44:21.832ZUpstairs last five completed: 1003 1002 1001 1000
4412025-01-23T17:44:21.832Z----------------------------------------------------------------
4422025-01-23T17:44:21.832Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4432025-01-23T17:44:21.832ZDownstairs last five completed: 1003 1002 1001 1000
4442025-01-23T17:44:21.832ZUpstairs last five completed: 1003 1002 1001 1000
4452025-01-23T17:44:21.835ZCLIENT: Up:0 ds:0 act:6
4462025-01-23T17:44:21.835ZCLIENT: All crucible jobs finished, exiting program
4472025-01-23T17:44:21.835Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:20.524760765Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1238}
4482025-01-23T17:44:21.839ZRepair part 1 passed
4492025-01-23T17:44:21.842Z
4502025-01-23T17:44:21.842ZKill the current downstairs
4512025-01-23T17:44:22.015ZDownstairs 2 stopped
4522025-01-23T17:44:22.015Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
4532025-01-23T17:44:22.018ZNow put back the original so we have a mismatch
4542025-01-23T17:44:22.018Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
4552025-01-23T17:44:22.021ZRestart downstairs with old directory
4562025-01-23T17:44:22.078Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4572025-01-23T17:44:22.095ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4582025-01-23T17:44:22.098Z 0 000-009 32 32 31 3996 3996 3995 F F F <---
4592025-01-23T17:44:22.098Z 1 010-019 32 32 31 3996 3996 3995 F F F <---
4602025-01-23T17:44:22.098Z 2 020-029 32 32 31 3996 3996 3995 F F F <---
4612025-01-23T17:44:22.098Z 3 030-039 32 32 31 3996 3996 3995 F F F <---
4622025-01-23T17:44:22.098Z 4 040-049 32 32 31 3996 3996 3995 F F F <---
4632025-01-23T17:44:22.098ZMax gen: 32, Max flush: 3996
4642025-01-23T17:44:22.098ZError: Difference in extent metadata found!
4652025-01-23T17:44:22.101Zdump test found error as expected
4662025-01-23T17:44:22.101Z
4672025-01-23T17:44:22.101Z
4682025-01-23T17:44:22.101Z/var/tmp/bins/crutest verify --range -g 33 -q --key ZkhwTgnB1LwVexb0z+8gydsSKERmT9kymvTEBkA7fYI= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
4692025-01-23T17:44:22.159Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.851083828Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"task":"crutest"}
4702025-01-23T17:44:22.162Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.851668463Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"task":"crutest"}
4712025-01-23T17:44:22.162Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.853312101Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"task":"crutest"}
4722025-01-23T17:44:22.162Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.853515311Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246}
4732025-01-23T17:44:22.162Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.853568801Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246}
4742025-01-23T17:44:22.162Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.853592766Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246}
4752025-01-23T17:44:22.162Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.853609183Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246}
4762025-01-23T17:44:22.162Z{"msg":"Crucible 34a4f7ae-82b4-4922-877d-dc0f047c09d1 has session id: 83f21ca1-0ca6-4e91-b8af-18fc5605a1f4","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.854309325Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
4772025-01-23T17:44:22.162Z{"msg":"Upstairs opts: Upstairs UUID: 34a4f7ae-82b4-4922-877d-dc0f047c09d1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.854373843Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
4782025-01-23T17:44:22.162Z{"msg":"Crucible stats registered with UUID: 34a4f7ae-82b4-4922-877d-dc0f047c09d1","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.854394039Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
4792025-01-23T17:44:22.162Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:20.854420364Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
4802025-01-23T17:44:22.165Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.856364322Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"task":"crutest"}
4812025-01-23T17:44:22.165Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.856468242Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246}
4822025-01-23T17:44:22.165Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.856504635Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246}
4832025-01-23T17:44:22.165Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.856517352Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246}
4842025-01-23T17:44:22.165Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.8565327Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246}
4852025-01-23T17:44:22.165Z{"msg":"Crucible 34a4f7ae-82b4-4922-877d-dc0f047c09d1 has session id: d2364668-9369-4597-b02c-17d9436c4f79","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.856895369Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
4862025-01-23T17:44:22.165Z{"msg":"Upstairs opts: Upstairs UUID: 34a4f7ae-82b4-4922-877d-dc0f047c09d1, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.856949288Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
4872025-01-23T17:44:22.165Z{"msg":"Crucible stats registered with UUID: 34a4f7ae-82b4-4922-877d-dc0f047c09d1","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.856972683Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
4882025-01-23T17:44:22.165Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:20.85698994Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
4892025-01-23T17:44:22.166Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.857707279Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"task":"crutest"}
4902025-01-23T17:44:22.166Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.857740053Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246}
4912025-01-23T17:44:22.166Z{"msg":"34a4f7ae-82b4-4922-877d-dc0f047c09d1 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.857842073Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
4922025-01-23T17:44:22.166Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.857880775Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"0","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
4932025-01-23T17:44:22.166Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.857920308Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"0","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
4942025-01-23T17:44:22.166Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.857936404Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"1","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
4952025-01-23T17:44:22.166Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.857965529Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"1","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
4962025-01-23T17:44:22.166Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.858015139Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"2","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
4972025-01-23T17:44:22.169Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.858036105Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"2","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
4982025-01-23T17:44:22.169Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.858072268Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
4992025-01-23T17:44:22.169Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.858102302Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5002025-01-23T17:44:22.169Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.858141654Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5012025-01-23T17:44:22.169Z{"msg":"ds_connection connected from Ok(127.0.0.1:38421)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.858256172Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5022025-01-23T17:44:22.169Z{"msg":"ds_connection connected from Ok(127.0.0.1:50224)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.858320099Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5032025-01-23T17:44:22.169Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:20.858421879Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5042025-01-23T17:44:22.169Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:20.858462961Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5052025-01-23T17:44:22.169Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:20.858489186Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5062025-01-23T17:44:22.169Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.858549564Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5072025-01-23T17:44:22.169Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.858866662Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"1","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5082025-01-23T17:44:22.169Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.858909923Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"0","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5092025-01-23T17:44:22.170Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.8597454Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"1","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5102025-01-23T17:44:22.170Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.859877234Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"0","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5112025-01-23T17:44:22.170Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.860098331Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5122025-01-23T17:44:22.170Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:20.860177435Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5132025-01-23T17:44:32.169Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.859615105Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5142025-01-23T17:44:32.174Z{"msg":"ds_connection connected from Ok(127.0.0.1:36742)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.859756237Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5152025-01-23T17:44:32.174Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.860696683Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"2","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5162025-01-23T17:44:32.174Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.861247125Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"2","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5172025-01-23T17:44:32.174Z{"msg":"[0]R flush_numbers: [3996, 3996, 3996, 3996, 3996]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.86163068Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"downstairs","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5182025-01-23T17:44:32.174Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.861665423Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"downstairs","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5192025-01-23T17:44:32.174Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.86168152Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"downstairs","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5202025-01-23T17:44:32.174Z{"msg":"[1]R flush_numbers: [3996, 3996, 3996, 3996, 3996]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.861704116Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"downstairs","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5212025-01-23T17:44:32.174Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.861718553Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"downstairs","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5222025-01-23T17:44:32.174Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.86173213Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"downstairs","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5232025-01-23T17:44:32.174Z{"msg":"[2]R flush_numbers: [3995, 3995, 3995, 3995, 3995]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.861752566Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"downstairs","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5242025-01-23T17:44:32.174Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.861766433Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"downstairs","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5252025-01-23T17:44:32.174Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.86178067Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"downstairs","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5262025-01-23T17:44:32.174Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.861793328Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"downstairs","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5272025-01-23T17:44:32.174Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.861806256Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"downstairs","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5282025-01-23T17:44:32.174Z{"msg":"Next flush: 3997","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.861819183Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"downstairs","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5292025-01-23T17:44:32.174Z{"msg":"extent 0 needs reconciliation: ExtentMetadata { gen: 32, flush: 3996, dirty: false } ExtentMetadata { gen: 32, flush: 3996, dirty: false } ExtentMetadata { gen: 31, flush: 3995, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.861841838Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"mend","":"downstairs","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5302025-01-23T17:44:32.174Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 3996, dirty: false }, ExtentMetadata { gen: 32, flush: 3996, dirty: false }, ExtentMetadata { gen: 31, flush: 3995, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.861858175Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"mend","":"downstairs","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5312025-01-23T17:44:32.174Z{"msg":"found dest for source 0 for extent at index 0 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.861878791Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"mend","":"downstairs","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5322025-01-23T17:44:32.174Z{"msg":"extent 1 needs reconciliation: ExtentMetadata { gen: 32, flush: 3996, dirty: false } ExtentMetadata { gen: 32, flush: 3996, dirty: false } ExtentMetadata { gen: 31, flush: 3995, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.861894258Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"mend","":"downstairs","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5332025-01-23T17:44:32.175Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 3996, dirty: false }, ExtentMetadata { gen: 32, flush: 3996, dirty: false }, ExtentMetadata { gen: 31, flush: 3995, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.861915924Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"mend","":"downstairs","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5342025-01-23T17:44:32.175Z{"msg":"found dest for source 0 for extent at index 1 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.861929931Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"mend","":"downstairs","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5352025-01-23T17:44:32.175Z{"msg":"extent 2 needs reconciliation: ExtentMetadata { gen: 32, flush: 3996, dirty: false } ExtentMetadata { gen: 32, flush: 3996, dirty: false } ExtentMetadata { gen: 31, flush: 3995, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.861952447Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"mend","":"downstairs","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5362025-01-23T17:44:32.175Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 3996, dirty: false }, ExtentMetadata { gen: 32, flush: 3996, dirty: false }, ExtentMetadata { gen: 31, flush: 3995, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.861968014Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"mend","":"downstairs","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5372025-01-23T17:44:32.175Z{"msg":"found dest for source 0 for extent at index 2 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.86198843Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"mend","":"downstairs","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5382025-01-23T17:44:32.175Z{"msg":"extent 3 needs reconciliation: ExtentMetadata { gen: 32, flush: 3996, dirty: false } ExtentMetadata { gen: 32, flush: 3996, dirty: false } ExtentMetadata { gen: 31, flush: 3995, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.862004407Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"mend","":"downstairs","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5392025-01-23T17:44:32.175Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 3996, dirty: false }, ExtentMetadata { gen: 32, flush: 3996, dirty: false }, ExtentMetadata { gen: 31, flush: 3995, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.862026872Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"mend","":"downstairs","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5402025-01-23T17:44:32.175Z{"msg":"found dest for source 0 for extent at index 3 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.862041099Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"mend","":"downstairs","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5412025-01-23T17:44:32.175Z{"msg":"extent 4 needs reconciliation: ExtentMetadata { gen: 32, flush: 3996, dirty: false } ExtentMetadata { gen: 32, flush: 3996, dirty: false } ExtentMetadata { gen: 31, flush: 3995, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.862062715Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"mend","":"downstairs","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5422025-01-23T17:44:32.175Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 3996, dirty: false }, ExtentMetadata { gen: 32, flush: 3996, dirty: false }, ExtentMetadata { gen: 31, flush: 3995, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.862077872Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"mend","":"downstairs","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5432025-01-23T17:44:32.175Z{"msg":"found dest for source 0 for extent at index 4 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.862097808Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"mend","":"downstairs","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5442025-01-23T17:44:32.175Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.862119224Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"0","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5452025-01-23T17:44:32.175Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.86214105Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"1","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5462025-01-23T17:44:32.175Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.862161316Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"2","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5472025-01-23T17:44:32.175Z{"msg":"Full repair list: {ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.862178992Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"downstairs","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5482025-01-23T17:44:32.175Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 3997, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 3997, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 3997, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 3997, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 3997, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.862318285Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"downstairs","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5492025-01-23T17:44:32.176Z{"msg":"starting reconciliation 7862c504-5f2e-4cda-9b86-b02cbfcb368e: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.862357008Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"downstairs","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5502025-01-23T17:44:32.176Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.86255072Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"2","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5512025-01-23T17:44:32.181Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.871132417Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"2","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5522025-01-23T17:44:32.184Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.874803928Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"2","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5532025-01-23T17:44:32.188Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.878576098Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"2","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5542025-01-23T17:44:32.192Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.882243439Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"2","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5552025-01-23T17:44:32.195Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.885359079Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"downstairs","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5562025-01-23T17:44:32.195Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.885400831Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"0","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5572025-01-23T17:44:32.195Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.885421407Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"1","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5582025-01-23T17:44:32.198Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.885447771Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"2","":"downstairs","session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5592025-01-23T17:44:32.198Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.885473386Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5602025-01-23T17:44:32.198Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.885494972Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5612025-01-23T17:44:32.198Z{"msg":"34a4f7ae-82b4-4922-877d-dc0f047c09d1 is now active with session: 83f21ca1-0ca6-4e91-b8af-18fc5605a1f4","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.885530705Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"session_id":"83f21ca1-0ca6-4e91-b8af-18fc5605a1f4"}
5622025-01-23T17:44:32.198Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.885549181Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246}
5632025-01-23T17:44:32.198Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.885563119Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246}
5642025-01-23T17:44:32.198Z{"msg":"34a4f7ae-82b4-4922-877d-dc0f047c09d1 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.885586704Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
5652025-01-23T17:44:32.198Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.885602311Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"0","":"downstairs","session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
5662025-01-23T17:44:32.198Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.885619608Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"0","":"downstairs","session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
5672025-01-23T17:44:32.198Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.885635994Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"1","":"downstairs","session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
5682025-01-23T17:44:32.198Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.885661439Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"1","":"downstairs","session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
5692025-01-23T17:44:32.198Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.885701461Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"2","":"downstairs","session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
5702025-01-23T17:44:32.198Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.885733605Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"2","":"downstairs","session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
5712025-01-23T17:44:32.198Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.88576314Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
5722025-01-23T17:44:32.198Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.885793123Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
5732025-01-23T17:44:32.198Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.885824877Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
5742025-01-23T17:44:32.198Z{"msg":"ds_connection connected from Ok(127.0.0.1:62765)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.885858111Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
5752025-01-23T17:44:32.198Z{"msg":"ds_connection connected from Ok(127.0.0.1:52656)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.885891734Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
5762025-01-23T17:44:32.198Z{"msg":"ds_connection connected from Ok(127.0.0.1:57448)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.885924478Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
5772025-01-23T17:44:32.198Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.886110541Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"2","":"downstairs","session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
5782025-01-23T17:44:32.198Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.886199394Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"1","":"downstairs","session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
5792025-01-23T17:44:32.198Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.886286687Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"0","":"downstairs","session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
5802025-01-23T17:44:32.199Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.886656254Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"2","":"downstairs","session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
5812025-01-23T17:44:32.199Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.886707534Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"1","":"downstairs","session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
5822025-01-23T17:44:32.199Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.88683288Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"0","":"downstairs","session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
5832025-01-23T17:44:32.199Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.887001347Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
5842025-01-23T17:44:32.199Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.887114684Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
5852025-01-23T17:44:32.199Z{"msg":"[0]R flush_numbers: [5318, 5318, 5318, 5318, 5318]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.887199078Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"downstairs","":"downstairs","session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
5862025-01-23T17:44:32.199Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.887226982Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"downstairs","":"downstairs","session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
5872025-01-23T17:44:32.199Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.88724168Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"downstairs","":"downstairs","session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
5882025-01-23T17:44:32.199Z{"msg":"[1]R flush_numbers: [5318, 5318, 5318, 5318, 5318]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.887267485Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"downstairs","":"downstairs","session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
5892025-01-23T17:44:32.199Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.887281532Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"downstairs","":"downstairs","session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
5902025-01-23T17:44:32.199Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.887295369Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"downstairs","":"downstairs","session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
5912025-01-23T17:44:32.199Z{"msg":"[2]R flush_numbers: [5318, 5318, 5318, 5318, 5318]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.887309806Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"downstairs","":"downstairs","session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
5922025-01-23T17:44:32.199Z{"msg":"Wait for a query_work_queue command to finish before sending IO
5932025-01-23T17:44:32.199Z[2]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.887336471Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"downstairs","":"downstairs","session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
5942025-01-23T17:44:32.199Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.887349778Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"downstairs","":"downstairs","session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
5952025-01-23T17:44:32.199Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.887368795Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"downstairs","":"downstairs","session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
5962025-01-23T17:44:32.200Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.887381902Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"downstairs","":"downstairs","session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
5972025-01-23T17:44:32.200Z{"msg":"Next flush: 5319","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.887400928Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"downstairs","":"downstairs","session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
5982025-01-23T17:44:32.200Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.887413956Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"":"downstairs","":"downstairs","session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
5992025-01-23T17:44:32.200Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.887426643Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
6002025-01-23T17:44:32.200Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.88744479Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"0","":"downstairs","session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
6012025-01-23T17:44:32.200Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.887457837Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"1","":"downstairs","session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
6022025-01-23T17:44:32.200Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.887477133Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"client":"2","":"downstairs","session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
6032025-01-23T17:44:32.200Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.887490101Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
6042025-01-23T17:44:32.200Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.887502708Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
6052025-01-23T17:44:32.200Z{"msg":"34a4f7ae-82b4-4922-877d-dc0f047c09d1 is now active with session: d2364668-9369-4597-b02c-17d9436c4f79","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.887522185Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
6062025-01-23T17:44:32.200Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.887533962Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246,"session_id":"d2364668-9369-4597-b02c-17d9436c4f79"}
6072025-01-23T17:44:32.201Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:30.887552539Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246}
6082025-01-23T17:44:32.201ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
6092025-01-23T17:44:32.201ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
6102025-01-23T17:44:32.201ZRead and Verify all blocks (0..100 range:true)
6112025-01-23T17:44:32.276ZWrote out file "/var/tmp/test_up-build/verify_file"
6122025-01-23T17:44:32.276ZVerify test completed
6132025-01-23T17:44:32.279ZWrote out file "/var/tmp/test_up-build/verify_file"
6142025-01-23T17:44:32.279ZCLIENT: Tests done. All submitted work has been ACK'd
6152025-01-23T17:44:32.279Z----------------------------------------------------------------
6162025-01-23T17:44:32.279Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6172025-01-23T17:44:32.279ZStates: Active Active Active
6182025-01-23T17:44:32.279ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6192025-01-23T17:44:32.279Z 1000 Acked Read 50 Done Done Done false
6202025-01-23T17:44:32.279Z STATES DS:0 DS:1 DS:2 TOTAL
6212025-01-23T17:44:32.279Z Sent 0 0 0 0
6222025-01-23T17:44:32.279Z Done 1 1 1 3
6232025-01-23T17:44:32.279Z Skipped 0 0 0 0
6242025-01-23T17:44:32.279Z Error 0 0 0 0
6252025-01-23T17:44:32.279ZLast Flush: None None None
6262025-01-23T17:44:32.279ZDownstairs last five completed:
6272025-01-23T17:44:32.279ZUpstairs last five completed: 1000
6282025-01-23T17:44:32.279Z----------------------------------------------------------------
6292025-01-23T17:44:32.279Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6302025-01-23T17:44:32.280ZStates: Active Active Active
6312025-01-23T17:44:32.280ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6322025-01-23T17:44:32.280Z 1000 Acked Read 50 Done Done Done false
6332025-01-23T17:44:32.280Z STATES DS:0 DS:1 DS:2 TOTAL
6342025-01-23T17:44:32.280Z Sent 0 0 0 0
6352025-01-23T17:44:32.280Z Done 1 1 1 3
6362025-01-23T17:44:32.280Z Skipped 0 0 0 0
6372025-01-23T17:44:32.280Z Error 0 0 0 0
6382025-01-23T17:44:32.280ZLast Flush: None None None
6392025-01-23T17:44:32.280ZDownstairs last five completed:
6402025-01-23T17:44:32.280ZUpstairs last five completed: 1000
6412025-01-23T17:44:32.280ZCLIENT: Up:0 ds:2 act:6
6422025-01-23T17:44:36.279Z----------------------------------------------------------------
6432025-01-23T17:44:36.279Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6442025-01-23T17:44:36.279ZDownstairs last five completed: 1001 1000
6452025-01-23T17:44:36.279ZUpstairs last five completed: 1001 1000
6462025-01-23T17:44:36.283Z----------------------------------------------------------------
6472025-01-23T17:44:36.283Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6482025-01-23T17:44:36.283ZDownstairs last five completed: 1001 1000
6492025-01-23T17:44:36.283ZUpstairs last five completed: 1001 1000
6502025-01-23T17:44:36.283ZCLIENT: Up:0 ds:0 act:6
6512025-01-23T17:44:36.283ZCLIENT: All crucible jobs finished, exiting program
6522025-01-23T17:44:36.283Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:34.968672754Z","hostname":"w-01JJA3RXZDDZM3TEBFC9AVDATJ","pid":1246}
6532025-01-23T17:44:36.286ZRepair part 2 passed
6542025-01-23T17:44:36.289Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
6552025-01-23T17:44:36.304ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6562025-01-23T17:44:36.304Z 0 000-009 32 32 32 3996 3996 3996 F F F
6572025-01-23T17:44:36.304Z 1 010-019 32 32 32 3996 3996 3996 F F F
6582025-01-23T17:44:36.304Z 2 020-029 32 32 32 3996 3996 3996 F F F
6592025-01-23T17:44:36.307Z 3 030-039 32 32 32 3996 3996 3996 F F F
6602025-01-23T17:44:36.307Z 4 040-049 32 32 32 3996 3996 3996 F F F
6612025-01-23T17:44:36.307ZMax gen: 32, Max flush: 3996
6622025-01-23T17:44:36.310Zdump test passed
6632025-01-23T17:44:36.310Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
6642025-01-23T17:44:36.326Z Extent 1
6652025-01-23T17:44:36.329ZGEN 32 32 32
6662025-01-23T17:44:36.329ZFLUSH_ID 3996 3996 3996
6672025-01-23T17:44:36.329ZDIRTY
6682025-01-23T17:44:36.329Z
6692025-01-23T17:44:36.329ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
6702025-01-23T17:44:36.329Z010 A A A A A A
6712025-01-23T17:44:36.332Z011 A A A A A A
6722025-01-23T17:44:36.332Z012 A A A A A A
6732025-01-23T17:44:36.334Z013 A A A A A A
6742025-01-23T17:44:36.334Z014 A A A A A A
6752025-01-23T17:44:36.337Z015 A A A A A A
6762025-01-23T17:44:36.337Z016 A A A A A A
6772025-01-23T17:44:36.340Z017 A A A A A A
6782025-01-23T17:44:36.340Z018 A A A A A A
6792025-01-23T17:44:36.340Z019 A A A A A A
6802025-01-23T17:44:36.344Zdump extent test passed
6812025-01-23T17:44:36.347Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
6822025-01-23T17:44:36.361ZExtent 2 Block in extent 0 Actual block 20
6832025-01-23T17:44:36.364Z
6842025-01-23T17:44:36.364Z DATA SHA256 VER
6852025-01-23T17:44:36.364Z------ ---------------------------------------------------------------- ---
6862025-01-23T17:44:36.364Z 0 effcade60de829c6aa8e7242587c0c3b6a07d0fcde1df1e153947e32cea8c271 A
6872025-01-23T17:44:36.367Z 1 effcade60de829c6aa8e7242587c0c3b6a07d0fcde1df1e153947e32cea8c271 A
6882025-01-23T17:44:36.367Z 2 effcade60de829c6aa8e7242587c0c3b6a07d0fcde1df1e153947e32cea8c271 A
6892025-01-23T17:44:36.367Z
6902025-01-23T17:44:36.367ZNONCES 0 1 2 DIFF
6912025-01-23T17:44:36.367Z------ ------------------------ ------------------------ ------------------------ -----
6922025-01-23T17:44:36.367Z 0 e52ccc7db2a888947be1a7bb e52ccc7db2a888947be1a7bb e52ccc7db2a888947be1a7bb
6932025-01-23T17:44:36.367Z
6942025-01-23T17:44:36.367Z TAGS 0 1 2 DIFF
6952025-01-23T17:44:36.367Z------ -------------------------------- -------------------------------- -------------------------------- -----
6962025-01-23T17:44:36.367Z 0 5a40e54678b28ff8a180a4d25dfbd1c0 5a40e54678b28ff8a180a4d25dfbd1c0 5a40e54678b28ff8a180a4d25dfbd1c0
6972025-01-23T17:44:36.367Z
6982025-01-23T17:44:36.367ZHASHES 0 1 2 DIFF
6992025-01-23T17:44:36.367Z------ ---------------- ---------------- ---------------- -----
7002025-01-23T17:44:36.367Z
7012025-01-23T17:44:36.370Zdump block test passed
7022025-01-23T17:44:36.370ZInitial upstairs tests have completed, stopping all downstairs
7032025-01-23T17:44:41.435ZCreating 7 larger downstairs regions
7042025-01-23T17:44:41.622ZStarting 7 downstairs
7052025-01-23T17:44:46.630Zdsc restarted at PID: 1262
7062025-01-23T17:44:47.029ZNow do the replace-reconcile test
7072025-01-23T17:44:47.085ZUsing 8870 for the replacement port
7082025-01-23T17:47:48.325ZCompleted test: replace-reconcile
7092025-01-23T17:47:48.325ZNow do the replace-before-active test
7102025-01-23T17:51:23.099ZCompleted test: replace-before-active
7112025-01-23T17:51:23.102ZAll tests have completed, stopping all downstairs
7122025-01-23T17:51:23.160Z
7132025-01-23T17:51:23.160ZAll Tests have passed
7142025-01-23T17:51:23.163Z13:41 Test duration
7152025-01-23T17:51:23.163Z
7162025-01-23T17:51:23.163Zreal 13:41.564649443
7172025-01-23T17:51:23.163Zuser 18:33.217710835
7182025-01-23T17:51:23.163Zsys 25.345936218
7192025-01-23T17:51:23.163Ztrap 0.541173250
7202025-01-23T17:51:23.163Ztflt 0.044519426
7212025-01-23T17:51:23.163Zdflt 0.052330168
7222025-01-23T17:51:23.163Zkflt 0.001514781
7232025-01-23T17:51:23.163Zlock 25:20:38.236107519
7242025-01-23T17:51:23.163Zslp 1:14:14.414724041
7252025-01-23T17:51:23.163Zlat 2:01.417138024
7262025-01-23T17:51:23.163Zstop 0.199339500
7272025-01-23T17:51:23.163Z+ echo 'test-up-2region-encrypted ends'
7282025-01-23T17:51:23.163Ztest-up-2region-encrypted ends
7292025-01-23T17:51:28.168Zprocess exited: duration 829891 ms, exit code 0
7302025-01-23T17:51:28.168Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
7312025-01-23T17:52:28.218Zexec warning: : stdout descriptor may be held open by a background process; giving up!
7322025-01-23T17:52:28.218Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
7332025-01-23T17:52:28.229Zfound 17 output files
7342025-01-23T17:52:28.230Zuploading: /tmp/test_up-build/dsc-out.txt (51348 bytes)
7352025-01-23T17:52:29.244Zuploaded: /tmp/test_up-build/dsc-out.txt
7362025-01-23T17:52:29.250Zuploading: /tmp/test_up-build/test_up_out.txt (8321556 bytes)
7372025-01-23T17:52:30.405Zuploaded: /tmp/test_up-build/test_up_out.txt
7382025-01-23T17:52:30.405Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (147274 bytes)
7392025-01-23T17:52:31.424Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
7402025-01-23T17:52:31.427Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (144659 bytes)
7412025-01-23T17:52:32.444Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
7422025-01-23T17:52:32.448Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (251054 bytes)
7432025-01-23T17:52:33.467Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
7442025-01-23T17:52:33.472Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (139430 bytes)
7452025-01-23T17:52:34.486Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
7462025-01-23T17:52:34.489Zuploading: /tmp/test_up-build/dsc/downstairs-8850.txt (2508 bytes)
7472025-01-23T17:52:35.495Zuploaded: /tmp/test_up-build/dsc/downstairs-8850.txt
7482025-01-23T17:52:35.498Zuploading: /tmp/test_up-build/dsc/downstairs-8860.txt (33795 bytes)
7492025-01-23T17:52:36.507Zuploaded: /tmp/test_up-build/dsc/downstairs-8860.txt
7502025-01-23T17:52:36.510Zuploading: /tmp/test_up-build/dsc/downstairs-8870.txt (842166 bytes)
7512025-01-23T17:52:37.548Zuploaded: /tmp/test_up-build/dsc/downstairs-8870.txt
7522025-01-23T17:52:37.551Zuploading: /tmp/debug/df.txt (1270 bytes)
7532025-01-23T17:52:38.557Zuploaded: /tmp/debug/df.txt
7542025-01-23T17:52:38.560Zuploading: /tmp/debug/dtrace.txt (979967 bytes)
7552025-01-23T17:52:39.602Zuploaded: /tmp/debug/dtrace.txt
7562025-01-23T17:52:39.602Zuploading: /tmp/debug/iostat.txt (266162 bytes)
7572025-01-23T17:52:39.623Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 266162 -> 269762
7582025-01-23T17:52:40.630Zuploaded: /tmp/debug/iostat.txt
7592025-01-23T17:52:40.632Zuploading: /tmp/debug/mpstat.txt (646638 bytes)
7602025-01-23T17:52:40.661Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 646638 -> 656115
7612025-01-23T17:52:41.668Zuploaded: /tmp/debug/mpstat.txt
7622025-01-23T17:52:41.670Zuploading: /tmp/debug/paging.txt (116067 bytes)
7632025-01-23T17:52:41.675Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 116067 -> 117993
7642025-01-23T17:52:42.681Zuploaded: /tmp/debug/paging.txt
7652025-01-23T17:52:42.685Zuploading: /tmp/debug/prstat.txt (1259104 bytes)
7662025-01-23T17:52:42.703Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 1259104 -> 1277221
7672025-01-23T17:52:43.709Zuploaded: /tmp/debug/prstat.txt
7682025-01-23T17:52:43.712Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
7692025-01-23T17:52:44.719Zuploaded: /tmp/debug/psrinfo.txt
7702025-01-23T17:52:44.722Zuploading: /tmp/debug/upstairs-info.txt (181390 bytes)
7712025-01-23T17:52:45.732Zuploaded: /tmp/debug/upstairs-info.txt