01JBM7T50NK5KVYNHYGW5J53D3: test-up-unencrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01JBM7W25T6MF56C500DBZ4YDZ

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-11-01T16:24:50.586Zjob dependencies complete; ready to run (waiting for 14 m 23 s)
22024-11-01T16:38:47.064Zjob assigned to worker 01JBM9EJNYZG1HY4F23R1SSEZZ (queued for 13 m 56 s)
32024-11-01T16:38:54.571Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-11-01T16:38:55.686Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-11-01T16:38:55.686Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-11-01T16:38:56.769Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-11-01T16:38:56.769Zdownloading input: /input/build/work/bins/crutest.gz
82024-11-01T16:38:57.923Zdownloaded input: /input/build/work/bins/crutest.gz
92024-11-01T16:38:57.923Zdownloading input: /input/build/work/bins/dsc.gz
102024-11-01T16:38:58.621Zdownloaded input: /input/build/work/bins/dsc.gz
112024-11-01T16:38:58.624Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-11-01T16:38:58.640Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-11-01T16:38:58.641Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-11-01T16:38:58.658Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-11-01T16:38:58.659Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-11-01T16:38:58.681Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-11-01T16:38:58.681Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-11-01T16:38:58.706Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-11-01T16:38:58.706Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-11-01T16:38:58.724Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-11-01T16:38:58.724Zdownloading input: /input/build/work/scripts/test_up.sh
222024-11-01T16:38:58.741Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-11-01T16:38:58.741Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-11-01T16:38:58.757Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-11-01T16:38:58.760Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-11-01T16:38:59.699Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-11-01T16:38:59.699Zstarting task 0: "setup"
282024-11-01T16:38:59.709Z++ uname -s
292024-11-01T16:38:59.712Z+ kern=SunOS
302024-11-01T16:38:59.714Z+ case "$kern" in
312024-11-01T16:38:59.714Z+ [[ 0 == 12345 ]]
322024-11-01T16:38:59.714Z+ groupadd -g 12345 build
332024-11-01T16:38:59.717Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
342024-11-01T16:39:01.724Z+ zfs create -o mountpoint=/work rpool/work
352024-11-01T16:39:02.284Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
362024-11-01T16:39:02.287Z+ home_fs=zfs
372024-11-01T16:39:02.287Z+ [[ zfs == autofs ]]
382024-11-01T16:39:02.287Z+ mkdir -p /home/build
392024-11-01T16:39:02.289Z+ chown build:build /home/build /work
402024-11-01T16:39:04.290Z+ chmod 0700 /home/build /work
412024-11-01T16:39:04.293Zprocess exited: duration 4593 ms, exit code 0
 
422024-11-01T16:39:04.300Zstarting task 1: "authentication"
432024-11-01T16:39:04.317Zprocess exited: duration 17 ms, exit code 0
 
442024-11-01T16:39:04.324Zstarting task 2: "build"
452024-11-01T16:39:04.327Z+ banner cores
462024-11-01T16:39:04.348Z
472024-11-01T16:39:04.348Z #### #### ##### ###### ####
482024-11-01T16:39:04.348Z # # # # # # # #
492024-11-01T16:39:04.348Z # # # # # ##### ####
502024-11-01T16:39:04.348Z # # # ##### # #
512024-11-01T16:39:04.348Z # # # # # # # # #
522024-11-01T16:39:04.348Z #### #### # # ###### ####
532024-11-01T16:39:04.348Z
542024-11-01T16:39:04.350Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
552024-11-01T16:39:04.359Z+ echo 'input bins dir contains:'
562024-11-01T16:39:04.359Zinput bins dir contains:
572024-11-01T16:39:04.361Z+ ls -ltr /input/build/work/bins
582024-11-01T16:39:04.361Ztotal 761017
592024-11-01T16:39:04.361Z-rw-r--r-- 1 root root 108991184 Nov 1 16:38 crucible-downstairs.gz
602024-11-01T16:39:04.361Z-rw-r--r-- 1 root root 104639051 Nov 1 16:38 crucible-hammer.gz
612024-11-01T16:39:04.361Z-rw-r--r-- 1 root root 109428757 Nov 1 16:38 crutest.gz
622024-11-01T16:39:04.361Z-rw-r--r-- 1 root root 66334129 Nov 1 16:38 dsc.gz
632024-11-01T16:39:04.361Z+ banner unpack
642024-11-01T16:39:04.364Z
652024-11-01T16:39:04.364Z # # # # ##### ## #### # #
662024-11-01T16:39:04.364Z # # ## # # # # # # # # #
672024-11-01T16:39:04.364Z # # # # # # # # # # ####
682024-11-01T16:39:04.364Z # # # # # ##### ###### # # #
692024-11-01T16:39:04.364Z # # # ## # # # # # # #
702024-11-01T16:39:04.365Z #### # # # # # #### # #
712024-11-01T16:39:04.365Z
722024-11-01T16:39:04.365Z+ mkdir -p /var/tmp/bins
732024-11-01T16:39:04.367Z+ for t in "$input/bins/"*.gz
742024-11-01T16:39:04.368Z++ basename /input/build/work/bins/crucible-downstairs.gz
752024-11-01T16:39:04.368Z+ b=crucible-downstairs.gz
762024-11-01T16:39:04.368Z+ b=crucible-downstairs
772024-11-01T16:39:04.368Z+ gunzip
782024-11-01T16:39:06.606Z+ chmod +x /var/tmp/bins/crucible-downstairs
792024-11-01T16:39:06.609Z+ for t in "$input/bins/"*.gz
802024-11-01T16:39:06.612Z++ basename /input/build/work/bins/crucible-hammer.gz
812024-11-01T16:39:06.612Z+ b=crucible-hammer.gz
822024-11-01T16:39:06.612Z+ b=crucible-hammer
832024-11-01T16:39:06.612Z+ gunzip
842024-11-01T16:39:08.755Z+ chmod +x /var/tmp/bins/crucible-hammer
852024-11-01T16:39:08.757Z+ for t in "$input/bins/"*.gz
862024-11-01T16:39:08.760Z++ basename /input/build/work/bins/crutest.gz
872024-11-01T16:39:08.760Z+ b=crutest.gz
882024-11-01T16:39:08.760Z+ b=crutest
892024-11-01T16:39:08.760Z+ gunzip
902024-11-01T16:39:11.044Z+ chmod +x /var/tmp/bins/crutest
912024-11-01T16:39:11.047Z+ for t in "$input/bins/"*.gz
922024-11-01T16:39:11.050Z++ basename /input/build/work/bins/dsc.gz
932024-11-01T16:39:11.050Z+ b=dsc.gz
942024-11-01T16:39:11.050Z+ b=dsc
952024-11-01T16:39:11.050Z+ gunzip
962024-11-01T16:39:12.409Z+ chmod +x /var/tmp/bins/dsc
972024-11-01T16:39:12.411Z+ export BINDIR=/var/tmp/bins
982024-11-01T16:39:12.411Z+ BINDIR=/var/tmp/bins
992024-11-01T16:39:12.411Z+ jobpid=1139
1002024-11-01T16:39:12.414Z+ echo 'Setup debug logging'
1012024-11-01T16:39:12.414Z+ mkdir /tmp/debug
1022024-11-01T16:39:12.414ZSetup debug logging
1032024-11-01T16:39:12.414Z+ sleep 7200
1042024-11-01T16:39:12.414Z+ psrinfo -v
1052024-11-01T16:39:12.417Z+ df -h
1062024-11-01T16:39:12.442Z+ prstat -d d -mLc 1
1072024-11-01T16:39:12.444Z+ iostat -T d -xn 1
1082024-11-01T16:39:12.444Z+ mpstat -T d 1
1092024-11-01T16:39:12.444Z+ vmstat -T d -p 1
1102024-11-01T16:39:12.447Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1112024-11-01T16:39:12.447Z+ banner test_up_unencrypted
1122024-11-01T16:39:12.447Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1132024-11-01T16:39:12.450Z
1142024-11-01T16:39:12.450Z ##### ###### #### ##### # # ##### # # # #
1152024-11-01T16:39:12.450Z # # # # # # # # # # ## #
1162024-11-01T16:39:12.450Z # ##### #### # # # # # # # # # #
1172024-11-01T16:39:12.450Z # # # # # # ##### # # # # #
1182024-11-01T16:39:12.450Z # # # # # # # # # # # ##
1192024-11-01T16:39:12.450Z # ###### #### # ####### #### # ####### #### # #
1202024-11-01T16:39:12.451Z
1212024-11-01T16:39:12.451Z+ ptime -m bash /input/build/work/scripts/test_up.sh -N unencrypted
1222024-11-01T16:39:12.458Z/input/build/work
1232024-11-01T16:39:12.488ZTurn off color for downstairs dump
1242024-11-01T16:39:12.491Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1252024-11-01T16:39:12.491ZCreating 3 downstairs regions
1262024-11-01T16:39:12.839ZStarting 3 downstairs
1272024-11-01T16:39:12.842Zdsc started at PID: 1184
1282024-11-01T16:39:18.012ZDisable automatic restart on all downstairs
1292024-11-01T16:39:18.062Z
1302024-11-01T16:39:18.062ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1312024-11-01T16:39:18.066ZRunning test: span
1322024-11-01T16:39:22.238ZCompleted test: span
1332024-11-01T16:39:27.246ZRunning test: big
1342024-11-01T16:39:31.482ZCompleted test: big
1352024-11-01T16:39:36.490ZRunning test: dep
1362024-11-01T16:39:49.195ZCompleted test: dep
1372024-11-01T16:39:54.203ZRunning test: balloon
1382024-11-01T16:40:07.711ZCompleted test: balloon
1392024-11-01T16:40:12.719ZRunning test: deactivate
1402024-11-01T16:41:07.013ZCompleted test: deactivate
1412024-11-01T16:41:12.020ZRunning hammer
1422024-11-01T16:41:15.590ZRun repair tests
1432024-11-01T16:41:15.593Z/var/tmp/bins/crutest fill -g 31 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1442024-11-01T16:41:15.654Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.205786769Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"task":"crutest"}
1452024-11-01T16:41:15.657Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.206410557Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"task":"crutest"}
1462024-11-01T16:41:15.657Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.208123255Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"task":"crutest"}
1472024-11-01T16:41:15.657Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.208292557Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221}
1482024-11-01T16:41:15.657Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7796cc7e5b7a12d521784f7b6b2dd4b8dd6e6d5b\",\n git_commit_timestamp: \"2024-11-01T16:07:40.000000000Z\",\n git_branch: \"renovate/reqwest-0.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.208353817Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221}
1492024-11-01T16:41:15.657Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.208383192Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221}
1502024-11-01T16:41:15.657Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.208402489Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221}
1512024-11-01T16:41:15.657Z{"msg":"Crucible 1b80668d-cf1c-4729-a299-e432918aeaa1 has session id: e8d52876-9cd8-4265-adfc-88d8671ee4aa","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.208805773Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1522024-11-01T16:41:15.658Z{"msg":"Upstairs opts: Upstairs UUID: 1b80668d-cf1c-4729-a299-e432918aeaa1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.208865313Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1532024-11-01T16:41:15.658Z{"msg":"Crucible stats registered with UUID: 1b80668d-cf1c-4729-a299-e432918aeaa1","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.20888613Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1542024-11-01T16:41:15.691Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.242645229Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"task":"crutest"}
1552024-11-01T16:41:15.691Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.242703249Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221}
1562024-11-01T16:41:15.695Z{"msg":"1b80668d-cf1c-4729-a299-e432918aeaa1 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.242745372Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1572024-11-01T16:41:15.695Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.242787345Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"client":"0","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1582024-11-01T16:41:15.695Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.242846576Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"client":"0","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1592024-11-01T16:41:15.695Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.242906306Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"client":"1","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1602024-11-01T16:41:15.695Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.24294021Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"client":"1","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1612024-11-01T16:41:15.695Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.242986943Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"client":"2","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1622024-11-01T16:41:15.695Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.243171042Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"client":"2","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1632024-11-01T16:41:15.695Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.243290942Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"":"io task","client":"2","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1642024-11-01T16:41:15.695Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.243319558Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"":"io task","client":"0","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1652024-11-01T16:41:15.695Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.243356162Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"":"io task","client":"1","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1662024-11-01T16:41:15.695Z{"msg":"ds_connection connected from Ok(127.0.0.1:60928)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.243384007Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"":"io task","client":"2","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1672024-11-01T16:41:15.695Z{"msg":"ds_connection connected from Ok(127.0.0.1:44469)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.243419552Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"":"io task","client":"1","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1682024-11-01T16:41:15.695Z{"msg":"ds_connection connected from Ok(127.0.0.1:39449)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.243447157Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"":"io task","client":"0","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1692024-11-01T16:41:15.695Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.24379312Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"client":"1","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1702024-11-01T16:41:15.695Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.243843412Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"client":"1","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1712024-11-01T16:41:15.695Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.243896363Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"client":"2","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1722024-11-01T16:41:15.696Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.243939326Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"client":"2","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1732024-11-01T16:41:15.696Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.243985778Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"client":"0","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1742024-11-01T16:41:15.696Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.244014404Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"client":"0","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1752024-11-01T16:41:15.696Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.24458429Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"client":"1","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1762024-11-01T16:41:15.696Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.244626343Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"client":"2","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1772024-11-01T16:41:15.696Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.244721427Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"client":"0","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1782024-11-01T16:41:15.696Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.245105734Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"client":"1","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1792024-11-01T16:41:15.696Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.245141538Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"client":"1","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1802024-11-01T16:41:15.696Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.245156226Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1812024-11-01T16:41:15.696Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.24519418Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"client":"0","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1822024-11-01T16:41:15.696Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.245220745Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"client":"0","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1832024-11-01T16:41:15.697Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.245244531Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1842024-11-01T16:41:15.697Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.245265968Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"client":"2","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1852024-11-01T16:41:15.697Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.245289994Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"client":"2","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1862024-11-01T16:41:15.697Z{"msg":"[0]R flush_numbers: [1533, 1533, 1533, 1533, 1533]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.245319179Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"":"downstairs","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1872024-11-01T16:41:15.697Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.245344465Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"":"downstairs","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1882024-11-01T16:41:15.697Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.24537227Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"":"downstairs","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1892024-11-01T16:41:15.697Z{"msg":"[1]R flush_numbers: [1533, 1533, 1533, 1533, 1533]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.245396536Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"":"downstairs","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1902024-11-01T16:41:15.697Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.245424502Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"":"downstairs","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1912024-11-01T16:41:15.697ZWait for a query_work_queue command to finish before sending IO
1922024-11-01T16:41:15.697Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.245463455Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"":"downstairs","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1932024-11-01T16:41:15.697Z{"msg":"[2]R flush_numbers: [1533, 1533, 1533, 1533, 1533]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.245489141Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"":"downstairs","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1942024-11-01T16:41:15.697Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.245516547Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"":"downstairs","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1952024-11-01T16:41:15.697Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.245540593Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"":"downstairs","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1962024-11-01T16:41:15.697Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.245567568Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"":"downstairs","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1972024-11-01T16:41:15.697Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.245591344Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"":"downstairs","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1982024-11-01T16:41:15.697Z{"msg":"Next flush: 1534","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.245623029Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"":"downstairs","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
1992024-11-01T16:41:15.697Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.245646675Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"":"downstairs","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
2002024-11-01T16:41:15.697ZRegion: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2012024-11-01T16:41:15.697Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.24567655Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
2022024-11-01T16:41:15.700Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Fill test
2032024-11-01T16:41:15.700Z,"time":"2024-11-01T16:41:15.245699037Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"client":"0","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
2042024-11-01T16:41:15.700Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.245727592Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"client":"1","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
2052024-11-01T16:41:15.700Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.245752968Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"client":"2","":"downstairs","session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
2062024-11-01T16:41:15.701Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.245780913Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
2072024-11-01T16:41:15.701Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.245803719Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
2082024-11-01T16:41:15.701Z{"msg":"1b80668d-cf1c-4729-a299-e432918aeaa1 is now active with session: e8d52876-9cd8-4265-adfc-88d8671ee4aa","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.245831695Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
2092024-11-01T16:41:15.701Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.245853841Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221,"session_id":"e8d52876-9cd8-4265-adfc-88d8671ee4aa"}
2102024-11-01T16:41:15.701Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:15.245880197Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1221}
2112024-11-01T16:41:15.704ZRead and Verify all blocks (0..50 range:false)
2122024-11-01T16:41:15.709ZWrote out file "/var/tmp/test_up-build/verify_file"
2132024-11-01T16:41:15.709ZCLIENT: Tests done. All submitted work has been ACK'd
2142024-11-01T16:41:15.712Z----------------------------------------------------------------
2152024-11-01T16:41:15.712Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2162024-11-01T16:41:15.712ZStates: Active Active Active
2172024-11-01T16:41:15.712ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2182024-11-01T16:41:15.713Z 1002 Acked Read 50 Done Done Done false
2192024-11-01T16:41:15.713Z STATES DS:0 DS:1 DS:2 TOTAL
2202024-11-01T16:41:15.713Z Sent 0 0 0 0
2212024-11-01T16:41:15.713Z Done 1 1 1 3
2222024-11-01T16:41:15.713Z Skipped 0 0 0 0
2232024-11-01T16:41:15.713Z Error 0 0 0 0
2242024-11-01T16:41:15.713ZLast Flush: 1001 1001 1001
2252024-11-01T16:41:15.713ZDownstairs last five completed: 1001 1000
2262024-11-01T16:41:15.713ZUpstairs last five completed: 1002 1001 1000
2272024-11-01T16:41:15.713ZCLIENT: Up:0 ds:1 act:3
2282024-11-01T16:41:19.711Z----------------------------------------------------------------
2292024-11-01T16:41:19.711Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2302024-11-01T16:41:19.712ZDownstairs last five completed: 1003 1002 1001 1000
2312024-11-01T16:41:19.712ZUpstairs last five completed: 1003 1002 1001 1000
2322024-11-01T16:41:19.712ZCLIENT: Up:0 ds:0 act:3
2332024-11-01T16:41:19.712ZCLIENT: All crucible jobs finished, exiting program
2342024-11-01T16:41:19.723ZRepair setup passed
2352024-11-01T16:41:19.727ZCopy the region for /var/tmp/test_up-build/8830
2362024-11-01T16:41:19.729Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2372024-11-01T16:41:19.732Z/var/tmp/bins/crutest fill -g 32 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2382024-11-01T16:41:19.789Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.33961458Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"task":"crutest"}
2392024-11-01T16:41:19.791Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.340186286Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"task":"crutest"}
2402024-11-01T16:41:19.791Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.341828996Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"task":"crutest"}
2412024-11-01T16:41:19.791Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.342015975Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227}
2422024-11-01T16:41:19.791Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7796cc7e5b7a12d521784f7b6b2dd4b8dd6e6d5b\",\n git_commit_timestamp: \"2024-11-01T16:07:40.000000000Z\",\n git_branch: \"renovate/reqwest-0.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.342068257Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227}
2432024-11-01T16:41:19.791Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.342092613Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227}
2442024-11-01T16:41:19.791Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.34210826Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227}
2452024-11-01T16:41:19.792Z{"msg":"Crucible 912de210-7326-490c-b5bf-d7c4e759cae6 has session id: 77570395-c97b-4c78-984b-f19a5756e32b","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.342495326Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2462024-11-01T16:41:19.792Z{"msg":"Upstairs opts: Upstairs UUID: 912de210-7326-490c-b5bf-d7c4e759cae6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.342549968Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2472024-11-01T16:41:19.792Z{"msg":"Crucible stats registered with UUID: 912de210-7326-490c-b5bf-d7c4e759cae6","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.342575133Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2482024-11-01T16:41:19.826Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.376722649Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"task":"crutest"}
2492024-11-01T16:41:19.826Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.376768311Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227}
2502024-11-01T16:41:19.826Z{"msg":"912de210-7326-490c-b5bf-d7c4e759cae6 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.376784219Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2512024-11-01T16:41:19.826Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.376797637Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"client":"0","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2522024-11-01T16:41:19.830Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.376819223Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"client":"0","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2532024-11-01T16:41:19.830Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.376847089Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"client":"1","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2542024-11-01T16:41:19.830Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.376866365Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"client":"1","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2552024-11-01T16:41:19.830Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.376884712Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"client":"2","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2562024-11-01T16:41:19.830Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.376926215Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"client":"2","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2572024-11-01T16:41:19.831Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.376951181Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"":"io task","client":"2","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2582024-11-01T16:41:19.831Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.376974857Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"":"io task","client":"0","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2592024-11-01T16:41:19.831Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.376998813Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"":"io task","client":"1","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2602024-11-01T16:41:19.831Z{"msg":"ds_connection connected from Ok(127.0.0.1:52935)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.37714021Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"":"io task","client":"0","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2612024-11-01T16:41:19.831Z{"msg":"ds_connection connected from Ok(127.0.0.1:52927)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.377174684Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"":"io task","client":"1","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2622024-11-01T16:41:19.831Z{"msg":"ds_connection connected from Ok(127.0.0.1:58093)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.37719904Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"":"io task","client":"2","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2632024-11-01T16:41:19.831Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.378047921Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"client":"0","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2642024-11-01T16:41:19.831Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.378085825Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"client":"0","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2652024-11-01T16:41:19.831Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.378101232Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"client":"1","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2662024-11-01T16:41:19.831Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.37811501Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"client":"1","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2672024-11-01T16:41:19.832Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.378136916Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"client":"2","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2682024-11-01T16:41:19.832Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.378165312Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"client":"2","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2692024-11-01T16:41:19.832Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.378693535Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"client":"0","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2702024-11-01T16:41:19.832Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.378834352Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"client":"1","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2712024-11-01T16:41:19.832Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.378882474Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"client":"2","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2722024-11-01T16:41:19.832Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.379071793Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"client":"0","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2732024-11-01T16:41:19.832Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.379126384Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"client":"0","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2742024-11-01T16:41:19.832Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.379183204Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2752024-11-01T16:41:19.832Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.379227157Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"client":"2","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2762024-11-01T16:41:19.832Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.37927035Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"client":"2","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2772024-11-01T16:41:19.833Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.379314133Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2782024-11-01T16:41:19.833Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.379350917Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"client":"1","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2792024-11-01T16:41:19.833Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.379377632Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"client":"1","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2802024-11-01T16:41:19.833Z{"msg":"[0]R flush_numbers: [1534, 1534, 1534, 1534, 1534]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.379411197Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"":"downstairs","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2812024-11-01T16:41:19.834Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.379435383Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"":"downstairs","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2822024-11-01T16:41:19.834Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.379471227Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"":"downstairs","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2832024-11-01T16:41:19.834Z{"msg":"[1]R flush_numbers: [1534, 1534, 1534, 1534, 1534]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.379508641Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"":"downstairs","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2842024-11-01T16:41:19.834Z{"msg":Wait for a query_work_queue command to finish before sending IO
2852024-11-01T16:41:19.834Z"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.379597226Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"":"downstairs","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2862024-11-01T16:41:19.834Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.379642509Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"":"downstairs","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2872024-11-01T16:41:19.834Z{"msg":"[2]R flush_numbers: [1534, 1534, 1534, 1534, 1534]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.379680563Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"":"downstairs","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2882024-11-01T16:41:19.834Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.379719626Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"":"downstairs","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2892024-11-01T16:41:19.834Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.37975679Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"":"downstairs","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2902024-11-01T16:41:19.834Z{Region: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2912024-11-01T16:41:19.834Z"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.379798973Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"":"downstairs","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2922024-11-01T16:41:19.834Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.37981653Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"":"downstairs","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2932024-11-01T16:41:19.834Z{"msg":"Next flush: 1535","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.379838017Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"":"downstairs","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2942024-11-01T16:41:19.834Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.379851615Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"":"downstairs","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2952024-11-01T16:41:19.834Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.379865592Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2962024-11-01T16:41:19.834Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.379885299Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"client":"0","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2972024-11-01T16:41:19.834Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.379899407Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"client":"1","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2982024-11-01T16:41:19.835Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.379912604Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"client":"2","":"downstairs","session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
2992024-11-01T16:41:19.835Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.379932751Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
3002024-11-01T16:41:19.835Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.379945279Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
3012024-11-01T16:41:19.835Z{"msg":"912de210-7326-490c-b5bf-d7c4e759cae6 is now active with session: 77570395-c97b-4c78-984b-f19a5756e32b","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.379958207Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
3022024-11-01T16:41:19.835Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:19.379977414Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227,"session_id":"77570395-c97b-4c78-984b-f19a5756e32b"}
3032024-11-01T16:41:19.835Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30Loading write count information from file "/var/tmp/test_up-build/verify_file"
3042024-11-01T16:41:19.835Z,"time":"2024-11-01T16:41:19.379989882Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227}
3052024-11-01T16:41:19.835ZFill test
3062024-11-01T16:41:19.839ZRead and Verify all blocks (0..50 range:false)
3072024-11-01T16:41:19.845ZWrote out file "/var/tmp/test_up-build/verify_file"
3082024-11-01T16:41:19.846ZCLIENT: Tests done. All submitted work has been ACK'd
3092024-11-01T16:41:19.848Z----------------------------------------------------------------
3102024-11-01T16:41:19.848Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
3112024-11-01T16:41:19.848ZStates: Active Active Active
3122024-11-01T16:41:19.849ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
3132024-11-01T16:41:19.849Z 1002 Acked Read 50 Done Done Done false
3142024-11-01T16:41:19.849Z STATES DS:0 DS:1 DS:2 TOTAL
3152024-11-01T16:41:19.849Z Sent 0 0 0 0
3162024-11-01T16:41:19.849Z Done 1 1 1 3
3172024-11-01T16:41:19.850Z Skipped 0 0 0 0
3182024-11-01T16:41:19.850Z Error 0 0 0 0
3192024-11-01T16:41:19.850ZLast Flush: 1001 1001 1001
3202024-11-01T16:41:19.850ZDownstairs last five completed: 1001 1000
3212024-11-01T16:41:19.850ZUpstairs last five completed: 1002 1001 1000
3222024-11-01T16:41:19.850ZCLIENT: Up:0 ds:1 act:3
3232024-11-01T16:41:23.848Z----------------------------------------------------------------
3242024-11-01T16:41:23.848Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
3252024-11-01T16:41:23.848ZDownstairs last five completed: 1003 1002 1001 1000
3262024-11-01T16:41:23.848ZUpstairs last five completed: 1003 1002 1001 1000
3272024-11-01T16:41:23.850ZCLIENT: Up:0 ds:0 act:3
3282024-11-01T16:41:23.851ZCLIENT: All crucible jobs finished, exiting program
3292024-11-01T16:41:23.851Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:41:23.398534568Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1227}
3302024-11-01T16:41:23.855ZRepair part 1 passed
3312024-11-01T16:41:23.858Z
3322024-11-01T16:41:23.858ZKill the current downstairs
3332024-11-01T16:41:24.018ZDownstairs 2 stopped
3342024-11-01T16:41:24.018Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
3352024-11-01T16:41:24.021ZNow put back the original so we have a mismatch
3362024-11-01T16:41:24.021Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
3372024-11-01T16:41:24.023ZRestart downstairs with old directory
3382024-11-01T16:41:24.074Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
3392024-11-01T16:41:24.090ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3402024-11-01T16:41:24.093Z 0 000-009 32 32 31 1535 1535 1534 F F F <---
3412024-11-01T16:41:24.093Z 1 010-019 32 32 31 1535 1535 1534 F F F <---
3422024-11-01T16:41:24.093Z 2 020-029 32 32 31 1535 1535 1534 F F F <---
3432024-11-01T16:41:24.093Z 3 030-039 32 32 31 1535 1535 1534 F F F <---
3442024-11-01T16:41:24.094Z 4 040-049 32 32 31 1535 1535 1534 F F F <---
3452024-11-01T16:41:24.094ZMax gen: 32, Max flush: 1535
3462024-11-01T16:41:24.094ZError: Difference in extent metadata found!
3472024-11-01T16:41:24.096Zdump test found error as expected
3482024-11-01T16:41:24.096Z
3492024-11-01T16:41:24.096Z
3502024-11-01T16:41:24.096Z/var/tmp/bins/crutest verify --range -g 33 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
3512024-11-01T16:41:24.153Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.70286433Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"task":"crutest"}
3522024-11-01T16:41:24.155Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.703460342Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"task":"crutest"}
3532024-11-01T16:41:24.155Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.705149154Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"task":"crutest"}
3542024-11-01T16:41:24.155Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.705315217Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235}
3552024-11-01T16:41:24.155Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7796cc7e5b7a12d521784f7b6b2dd4b8dd6e6d5b\",\n git_commit_timestamp: \"2024-11-01T16:07:40.000000000Z\",\n git_branch: \"renovate/reqwest-0.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.70535874Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235}
3562024-11-01T16:41:24.155Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.705383696Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235}
3572024-11-01T16:41:24.155Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.705398924Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235}
3582024-11-01T16:41:24.162Z{"msg":"Crucible a00a69c2-f696-4c65-8098-6510e44006a7 has session id: 8ef3b353-4727-4a97-a2be-49f77efd49d1","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.705774552Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3592024-11-01T16:41:24.162Z{"msg":"Upstairs opts: Upstairs UUID: a00a69c2-f696-4c65-8098-6510e44006a7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.705826433Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3602024-11-01T16:41:24.162Z{"msg":"Crucible stats registered with UUID: a00a69c2-f696-4c65-8098-6510e44006a7","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.705851019Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3612024-11-01T16:41:24.189Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.739509295Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"task":"crutest"}
3622024-11-01T16:41:24.189Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.739570645Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235}
3632024-11-01T16:41:24.192Z{"msg":"a00a69c2-f696-4c65-8098-6510e44006a7 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.73960292Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3642024-11-01T16:41:24.192Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.739648952Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"client":"0","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3652024-11-01T16:41:24.192Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.73972485Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"client":"0","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3662024-11-01T16:41:24.192Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.739778831Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"client":"1","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3672024-11-01T16:41:24.192Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.739825163Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"client":"1","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3682024-11-01T16:41:24.193Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.739867226Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"client":"2","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3692024-11-01T16:41:24.193Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.739914658Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"client":"2","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3702024-11-01T16:41:24.193Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.739973579Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3712024-11-01T16:41:24.193Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.740050426Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"":"io task","client":"0","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3722024-11-01T16:41:24.193Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.740100088Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"":"io task","client":"1","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3732024-11-01T16:41:24.193Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:41:23.74014891Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3742024-11-01T16:41:24.193Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:41:23.740310443Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3752024-11-01T16:41:24.193Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:41:23.74038955Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3762024-11-01T16:41:24.193Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.740426654Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"client":"2","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3772024-11-01T16:41:24.193Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.74045484Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"client":"2","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3782024-11-01T16:41:24.193Z{"msg":"ds_connection connected from Ok(127.0.0.1:45340)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.740516799Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"":"io task","client":"0","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3792024-11-01T16:41:24.193Z{"msg":"ds_connection connected from Ok(127.0.0.1:42923)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.740550024Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"":"io task","client":"1","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3802024-11-01T16:41:24.193Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.740583768Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3812024-11-01T16:41:24.193Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.74081731Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"client":"1","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3822024-11-01T16:41:24.193Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.740867502Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"client":"1","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3832024-11-01T16:41:24.193Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.740909325Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"client":"0","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3842024-11-01T16:41:24.193Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.740946969Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"client":"0","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3852024-11-01T16:41:24.193Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.741455215Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"client":"0","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3862024-11-01T16:41:24.193Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.741496048Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"client":"1","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3872024-11-01T16:41:24.194Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.741873946Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"client":"0","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3882024-11-01T16:41:24.194Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.7419109Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"client":"0","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3892024-11-01T16:41:24.194Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.741926738Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3902024-11-01T16:41:24.194Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.741947554Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"client":"1","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3912024-11-01T16:41:24.194Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.741962012Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"client":"1","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3922024-11-01T16:41:24.194Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:23.741982409Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3932024-11-01T16:41:34.193Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.741348663Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3942024-11-01T16:41:34.193Z{"msg":"ds_connection connected from Ok(127.0.0.1:61973)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.74154569Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3952024-11-01T16:41:34.199Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.742484306Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"client":"2","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3962024-11-01T16:41:34.199Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.742523539Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"client":"2","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3972024-11-01T16:41:34.199Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.742972205Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"client":"2","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3982024-11-01T16:41:34.200Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.74331105Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"client":"2","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
3992024-11-01T16:41:34.200Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743343824Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"client":"2","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4002024-11-01T16:41:34.200Z{"msg":"[0]R flush_numbers: [1535, 1535, 1535, 1535, 1535]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743359572Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4012024-11-01T16:41:34.200Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743374769Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4022024-11-01T16:41:34.200Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743388287Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4032024-11-01T16:41:34.200Z{"msg":"[1]R flush_numbers: [1535, 1535, 1535, 1535, 1535]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743409493Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4042024-11-01T16:41:34.200Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743423321Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4052024-11-01T16:41:34.200Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743443728Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4062024-11-01T16:41:34.200Z{"msg":"[2]R flush_numbers: [1534, 1534, 1534, 1534, 1534]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743457866Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4072024-11-01T16:41:34.200Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743478462Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4082024-11-01T16:41:34.200Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.74349204Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4092024-11-01T16:41:34.200Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743512787Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4102024-11-01T16:41:34.200Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743526724Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4112024-11-01T16:41:34.200Z{"msg":"Next flush: 1536","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743547081Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4122024-11-01T16:41:34.200Z{"msg":"Extent 0 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743560579Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4132024-11-01T16:41:34.200Z{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743580915Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4142024-11-01T16:41:34.200Z{"msg":"extent:0 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743595793Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4152024-11-01T16:41:34.200Z{"msg":"extent:0 flush: 1535 1535 1534 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743617409Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4162024-11-01T16:41:34.200Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743632177Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4172024-11-01T16:41:34.200Z{"msg":"extent:0 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743653164Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4182024-11-01T16:41:34.200Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743668031Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4192024-11-01T16:41:34.200Z{"msg":"find dest for source 0 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743689658Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4202024-11-01T16:41:34.200Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743704675Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4212024-11-01T16:41:34.200Z{"msg":"Extent 1 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743744018Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4222024-11-01T16:41:34.200Z{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743766465Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4232024-11-01T16:41:34.200Z{"msg":"extent:1 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743780952Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4242024-11-01T16:41:34.200Z{"msg":"extent:1 flush: 1535 1535 1534 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743802429Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4252024-11-01T16:41:34.200Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743817097Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4262024-11-01T16:41:34.200Z{"msg":"extent:1 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743838353Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4272024-11-01T16:41:34.200Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743854Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4282024-11-01T16:41:34.201Z{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743875187Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4292024-11-01T16:41:34.201Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743889525Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4302024-11-01T16:41:34.201Z{"msg":"Extent 2 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743910771Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4312024-11-01T16:41:34.201Z{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743924929Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4322024-11-01T16:41:34.201Z{"msg":"extent:2 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743945705Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4332024-11-01T16:41:34.201Z{"msg":"extent:2 flush: 1535 1535 1534 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743960743Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4342024-11-01T16:41:34.201Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743981719Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4352024-11-01T16:41:34.201Z{"msg":"extent:2 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.743996147Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4362024-11-01T16:41:34.201Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.744017853Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4372024-11-01T16:41:34.201Z{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.744032521Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4382024-11-01T16:41:34.201Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.744053308Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4392024-11-01T16:41:34.201Z{"msg":"Extent 3 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.744068215Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4402024-11-01T16:41:34.201Z{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.744088372Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4412024-11-01T16:41:34.201Z{"msg":"extent:3 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.74410275Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4422024-11-01T16:41:34.201Z{"msg":"extent:3 flush: 1535 1535 1534 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.744124226Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4432024-11-01T16:41:34.201Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.744139124Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4442024-11-01T16:41:34.201Z{"msg":"extent:3 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.744166699Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4452024-11-01T16:41:34.201Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.744211602Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4462024-11-01T16:41:34.201Z{"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.744241527Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4472024-11-01T16:41:34.202Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.744267823Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4482024-11-01T16:41:34.202Z{"msg":"Extent 4 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.744293788Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4492024-11-01T16:41:34.202Z{"msg":"First source client ID for extent 4","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.744308346Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4502024-11-01T16:41:34.202Z{"msg":"extent:4 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.744322503Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4512024-11-01T16:41:34.202Z{"msg":"extent:4 flush: 1535 1535 1534 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.74434435Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4522024-11-01T16:41:34.202Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.744359217Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4532024-11-01T16:41:34.202Z{"msg":"extent:4 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.744374465Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4542024-11-01T16:41:34.202Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.744395671Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4552024-11-01T16:41:34.202Z{"msg":"find dest for source 0 for extent at index 4","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.744410139Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4562024-11-01T16:41:34.202Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.744430736Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4572024-11-01T16:41:34.202Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.744462091Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"client":"0","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4582024-11-01T16:41:34.202Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.744476718Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"client":"1","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4592024-11-01T16:41:34.202Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.744489726Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"client":"2","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4602024-11-01T16:41:34.202Z{"msg":"starting reconciliation 3b793c63-17ff-454b-9ff7-11896db57a7a: found 5 extents that need repair","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.744519801Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4612024-11-01T16:41:34.202Z{"msg":"Full repair list: {ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.744549676Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4622024-11-01T16:41:34.202Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 1536, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 1536, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 1536, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 1536, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 1536, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.744641681Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4632024-11-01T16:41:34.203Z{"msg":"no Nexus client from DNS, aborting start notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:41:33.744660408Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"reconcile":"3b793c63-17ff-454b-9ff7-11896db57a7a","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4642024-11-01T16:41:34.203Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:41:33.744676015Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"reconcile":"3b793c63-17ff-454b-9ff7-11896db57a7a","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4652024-11-01T16:41:34.203Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:41:33.744691183Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"reconcile":"3b793c63-17ff-454b-9ff7-11896db57a7a","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4662024-11-01T16:41:34.206Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.744840988Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"client":"2","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4672024-11-01T16:41:34.207Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:41:33.744924695Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"reconcile":"3b793c63-17ff-454b-9ff7-11896db57a7a","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4682024-11-01T16:41:34.211Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:41:33.759922329Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"reconcile":"3b793c63-17ff-454b-9ff7-11896db57a7a","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4692024-11-01T16:41:34.214Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:41:33.760353787Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"reconcile":"3b793c63-17ff-454b-9ff7-11896db57a7a","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4702024-11-01T16:41:34.214Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:41:33.760663347Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"reconcile":"3b793c63-17ff-454b-9ff7-11896db57a7a","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4712024-11-01T16:41:34.214Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.760958228Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"client":"2","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4722024-11-01T16:41:34.214Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:41:33.761002401Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"reconcile":"3b793c63-17ff-454b-9ff7-11896db57a7a","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4732024-11-01T16:41:34.222Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:41:33.770601683Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"reconcile":"3b793c63-17ff-454b-9ff7-11896db57a7a","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4742024-11-01T16:41:34.230Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:41:33.771006686Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"reconcile":"3b793c63-17ff-454b-9ff7-11896db57a7a","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4752024-11-01T16:41:34.230Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:41:33.771291029Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"reconcile":"3b793c63-17ff-454b-9ff7-11896db57a7a","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4762024-11-01T16:41:34.231Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.771581861Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"client":"2","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4772024-11-01T16:41:34.231Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:41:33.771622535Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"reconcile":"3b793c63-17ff-454b-9ff7-11896db57a7a","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4782024-11-01T16:41:34.234Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:41:33.781957785Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"reconcile":"3b793c63-17ff-454b-9ff7-11896db57a7a","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4792024-11-01T16:41:34.236Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:41:33.782356879Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"reconcile":"3b793c63-17ff-454b-9ff7-11896db57a7a","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4802024-11-01T16:41:34.236Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:41:33.782671727Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"reconcile":"3b793c63-17ff-454b-9ff7-11896db57a7a","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4812024-11-01T16:41:34.236Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.782947582Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"client":"2","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4822024-11-01T16:41:34.237Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:41:33.782995554Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"reconcile":"3b793c63-17ff-454b-9ff7-11896db57a7a","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4832024-11-01T16:41:34.244Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:41:33.792723175Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"reconcile":"3b793c63-17ff-454b-9ff7-11896db57a7a","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4842024-11-01T16:41:34.247Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:41:33.793203046Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"reconcile":"3b793c63-17ff-454b-9ff7-11896db57a7a","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4852024-11-01T16:41:34.247Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:41:33.7934809Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"reconcile":"3b793c63-17ff-454b-9ff7-11896db57a7a","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4862024-11-01T16:41:34.247Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.793738737Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"client":"2","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4872024-11-01T16:41:34.247Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:41:33.793793179Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"reconcile":"3b793c63-17ff-454b-9ff7-11896db57a7a","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4882024-11-01T16:41:34.254Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:41:33.802228852Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"reconcile":"3b793c63-17ff-454b-9ff7-11896db57a7a","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4892024-11-01T16:41:34.256Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.802584183Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4902024-11-01T16:41:34.256Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.802627586Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"client":"0","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4912024-11-01T16:41:34.256Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.802649762Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"client":"1","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4922024-11-01T16:41:34.256Z{"msg":"Transition from Reconcile to Active","Wait for a query_work_queue command to finish before sending IO
4932024-11-01T16:41:34.256Zv":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.802702554Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"client":"2","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4942024-11-01T16:41:34.256Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.802736028Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4952024-11-01T16:41:34.257Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.802765153Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4962024-11-01T16:41:34.257Z{"msg":"a00a69c2-f696-4c65-8098-6510e44006a7 is now active with session: 8ef3b353-4727-4a97-a2be-49f77efd49d1","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.802790469Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4972024-11-01T16:41:34.257Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:41:33.802827523Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235}
4982024-11-01T16:41:34.257Z{"msg":"no Nexus client from DNS, aborting finish notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:41:33.80284522Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235,"reconcile":"3b793c63-17ff-454b-9ff7-11896db57a7a","":"downstairs","":"downstairs","session_id":"8ef3b353-4727-4a97-a2be-49f77efd49d1"}
4992024-11-01T16:41:34.257ZRegion: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
5002024-11-01T16:41:34.257ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
5012024-11-01T16:41:34.257ZRead and Verify all blocks (0..50 range:true)
5022024-11-01T16:41:34.261ZWrote out file "/var/tmp/test_up-build/verify_file"
5032024-11-01T16:41:34.261ZVerify test completed
5042024-11-01T16:41:34.264ZWrote out file "/var/tmp/test_up-build/verify_file"
5052024-11-01T16:41:34.264ZCLIENT: Tests done. All submitted work has been ACK'd
5062024-11-01T16:41:34.264Z----------------------------------------------------------------
5072024-11-01T16:41:34.264Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
5082024-11-01T16:41:34.264ZStates: Active Active Active
5092024-11-01T16:41:34.264ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
5102024-11-01T16:41:34.264Z 1000 Acked Read 50 Done Done Done false
5112024-11-01T16:41:34.264Z STATES DS:0 DS:1 DS:2 TOTAL
5122024-11-01T16:41:34.264Z Sent 0 0 0 0
5132024-11-01T16:41:34.264Z Done 1 1 1 3
5142024-11-01T16:41:34.264Z Skipped 0 0 0 0
5152024-11-01T16:41:34.265Z Error 0 0 0 0
5162024-11-01T16:41:34.265ZLast Flush: 0 0 0
5172024-11-01T16:41:34.265ZDownstairs last five completed:
5182024-11-01T16:41:34.265ZUpstairs last five completed: 1000
5192024-11-01T16:41:34.265ZCLIENT: Up:0 ds:1 act:3
5202024-11-01T16:41:38.263Z----------------------------------------------------------------
5212024-11-01T16:41:38.263Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
5222024-11-01T16:41:38.263ZDownstairs last five completed: 1001 1000
5232024-11-01T16:41:38.263ZUpstairs last five completed: 1001 1000
5242024-11-01T16:41:38.263ZCLIENT: Up:0 ds:0 act:3
5252024-11-01T16:41:38.263ZCLIENT: All crucible jobs finished, exiting program
5262024-11-01T16:41:38.266Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:41:37.811138611Z","hostname":"w-01JBM9EJNYZG1HY4F23R1SSEZZ","pid":1235}
5272024-11-01T16:41:38.271ZRepair part 2 passed
5282024-11-01T16:41:38.273Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
5292024-11-01T16:41:38.287ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
5302024-11-01T16:41:38.288Z 0 000-009 32 32 32 1535 1535 1535 F F F
5312024-11-01T16:41:38.288Z 1 010-019 32 32 32 1535 1535 1535 F F F
5322024-11-01T16:41:38.288Z 2 020-029 32 32 32 1535 1535 1535 F F F
5332024-11-01T16:41:38.288Z 3 030-039 32 32 32 1535 1535 1535 F F F
5342024-11-01T16:41:38.290Z 4 040-049 32 32 32 1535 1535 1535 F F F
5352024-11-01T16:41:38.290ZMax gen: 32, Max flush: 1535
5362024-11-01T16:41:38.292Zdump test passed
5372024-11-01T16:41:38.293Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
5382024-11-01T16:41:38.306Z Extent 1
5392024-11-01T16:41:38.308ZGEN 32 32 32
5402024-11-01T16:41:38.308ZFLUSH_ID 1535 1535 1535
5412024-11-01T16:41:38.308ZDIRTY
5422024-11-01T16:41:38.308Z
5432024-11-01T16:41:38.308ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
5442024-11-01T16:41:38.308Z010 A A A A A A
5452024-11-01T16:41:38.311Z011 A A A A A A
5462024-11-01T16:41:38.311Z012 A A A A A A
5472024-11-01T16:41:38.311Z013 A A A A A A
5482024-11-01T16:41:38.313Z014 A A A A A A
5492024-11-01T16:41:38.314Z015 A A A A A A
5502024-11-01T16:41:38.316Z016 A A A A A A
5512024-11-01T16:41:38.316Z017 A A A A A A
5522024-11-01T16:41:38.319Z018 A A A A A A
5532024-11-01T16:41:38.319Z019 A A A A A A
5542024-11-01T16:41:38.321Zdump extent test passed
5552024-11-01T16:41:38.321Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
5562024-11-01T16:41:38.335ZExtent 2 Block in extent 0 Actual block 20
5572024-11-01T16:41:38.335Z
5582024-11-01T16:41:38.337Z DATA SHA256 VER
5592024-11-01T16:41:38.337Z------ ---------------------------------------------------------------- ---
5602024-11-01T16:41:38.337Z 0 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
5612024-11-01T16:41:38.338Z 1 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
5622024-11-01T16:41:38.338Z 2 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
5632024-11-01T16:41:38.338Z
5642024-11-01T16:41:38.338ZNONCES 0 1 2 DIFF
5652024-11-01T16:41:38.338Z------ ------------------------ ------------------------ ------------------------ -----
5662024-11-01T16:41:38.338Z
5672024-11-01T16:41:38.338Z TAGS 0 1 2 DIFF
5682024-11-01T16:41:38.340Z------ -------------------------------- -------------------------------- -------------------------------- -----
5692024-11-01T16:41:38.340Z
5702024-11-01T16:41:38.340ZHASHES 0 1 2 DIFF
5712024-11-01T16:41:38.340Z------ ---------------- ---------------- ---------------- -----
5722024-11-01T16:41:38.340Z 0 42916ab6fa364c70 42916ab6fa364c70 42916ab6fa364c70
5732024-11-01T16:41:38.340Z
5742024-11-01T16:41:38.340Zdump block test passed
5752024-11-01T16:41:38.340ZInitial upstairs tests have completed, stopping all downstairs
5762024-11-01T16:41:43.402ZCreating 4 larger downstairs regions
5772024-11-01T16:41:43.852ZStarting 4 downstairs
5782024-11-01T16:41:48.859Zdsc restarted at PID: 1248
5792024-11-01T16:41:49.073ZNow do the replace-reconcile test
5802024-11-01T16:41:49.125ZUsing 8840 for the replacement port
5812024-11-01T16:44:18.575ZCompleted test: replace-reconcile
5822024-11-01T16:44:18.578ZNow do the replace-before-active test
5832024-11-01T16:46:55.190ZCompleted test: replace-before-active
5842024-11-01T16:46:55.193ZAll tests have completed, stopping all downstairs
5852024-11-01T16:46:55.248Z
5862024-11-01T16:46:55.248ZAll Tests have passed
5872024-11-01T16:46:55.252Z7:42 Test duration
5882024-11-01T16:46:55.253Z
5892024-11-01T16:46:55.253Zreal 7:42.709649191
5902024-11-01T16:46:55.253Zuser 1:02.768348362
5912024-11-01T16:46:55.253Zsys 11.378680076
5922024-11-01T16:46:55.254Ztrap 0.141312306
5932024-11-01T16:46:55.254Ztflt 0.020614197
5942024-11-01T16:46:55.254Zdflt 0.050404388
5952024-11-01T16:46:55.254Zkflt 0.001296355
5962024-11-01T16:46:55.254Zlock 5:45:35.916988560
5972024-11-01T16:46:55.254Zslp 24:39.670134639
5982024-11-01T16:46:55.254Zlat 8.823250179
5992024-11-01T16:46:55.254Zstop 0.120973728
6002024-11-01T16:46:55.254Z+ echo 'test-up-unencrypted ends'
6012024-11-01T16:46:55.254Ztest-up-unencrypted ends
6022024-11-01T16:47:00.259Zprocess exited: duration 470838 ms, exit code 0
6032024-11-01T16:47:00.259Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
6042024-11-01T16:48:00.313Zexec warning: : stdout descriptor may be held open by a background process; giving up!
6052024-11-01T16:48:00.313Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
6062024-11-01T16:48:00.327Zfound 14 output files
6072024-11-01T16:48:00.327Zuploading: /tmp/test_up-build/dsc-out.txt (37037 bytes)
6082024-11-01T16:48:01.346Zuploaded: /tmp/test_up-build/dsc-out.txt
6092024-11-01T16:48:01.346Zuploading: /tmp/test_up-build/test_up_out.txt (10515859 bytes)
6102024-11-01T16:48:02.505Zuploaded: /tmp/test_up-build/test_up_out.txt
6112024-11-01T16:48:02.510Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (253694 bytes)
6122024-11-01T16:48:03.532Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
6132024-11-01T16:48:03.535Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (144653 bytes)
6142024-11-01T16:48:04.552Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
6152024-11-01T16:48:04.555Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (251120 bytes)
6162024-11-01T16:48:05.577Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
6172024-11-01T16:48:05.580Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (139434 bytes)
6182024-11-01T16:48:06.596Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
6192024-11-01T16:48:06.596Zuploading: /tmp/debug/df.txt (1270 bytes)
6202024-11-01T16:48:07.605Zuploaded: /tmp/debug/df.txt
6212024-11-01T16:48:07.605Zuploading: /tmp/debug/dtrace.txt (419510 bytes)
6222024-11-01T16:48:08.634Zuploaded: /tmp/debug/dtrace.txt
6232024-11-01T16:48:08.636Zuploading: /tmp/debug/iostat.txt (159006 bytes)
6242024-11-01T16:48:08.650Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 159006 -> 161715
6252024-11-01T16:48:09.656Zuploaded: /tmp/debug/iostat.txt
6262024-11-01T16:48:09.659Zuploading: /tmp/debug/mpstat.txt (385447 bytes)
6272024-11-01T16:48:09.668Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 385447 -> 392747
6282024-11-01T16:48:10.675Zuploaded: /tmp/debug/mpstat.txt
6292024-11-01T16:48:10.677Zuploading: /tmp/debug/paging.txt (69622 bytes)
6302024-11-01T16:48:10.682Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 69622 -> 71056
6312024-11-01T16:48:11.688Zuploaded: /tmp/debug/paging.txt
6322024-11-01T16:48:11.700Zuploading: /tmp/debug/prstat.txt (727571 bytes)
6332024-11-01T16:48:11.753Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 727571 -> 741825
6342024-11-01T16:48:12.763Zuploaded: /tmp/debug/prstat.txt
6352024-11-01T16:48:12.766Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
6362024-11-01T16:48:13.772Zuploaded: /tmp/debug/psrinfo.txt
6372024-11-01T16:48:13.774Zuploading: /tmp/debug/upstairs-info.txt (71100 bytes)
6382024-11-01T16:48:14.786Zuploaded: /tmp/debug/upstairs-info.txt