01JENYDE7Y45ACFA0EQMJ32BNB: test-up-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01JENYDTCGJ5WQDB43199J6NTR

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-12-09T15:06:24.998Zjob dependencies complete; ready to run (waiting for 15 m 27 s)
22024-12-09T15:07:21.391Zjob assigned to worker 01JENZAAS3HDWWSJJ20Q5Q6PP4 [factory aws, i-05f9d162b466fa37a] (queued for 56 s)
32024-12-09T15:07:30.085Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-12-09T15:07:31.294Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-12-09T15:07:31.294Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-12-09T15:07:32.438Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-12-09T15:07:32.438Zdownloading input: /input/build/work/bins/crutest.gz
82024-12-09T15:07:33.542Zdownloaded input: /input/build/work/bins/crutest.gz
92024-12-09T15:07:33.542Zdownloading input: /input/build/work/bins/dsc.gz
102024-12-09T15:07:34.261Zdownloaded input: /input/build/work/bins/dsc.gz
112024-12-09T15:07:34.266Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-12-09T15:07:34.282Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-12-09T15:07:34.282Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-12-09T15:07:34.303Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-12-09T15:07:34.303Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-12-09T15:07:34.323Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-12-09T15:07:34.323Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-12-09T15:07:34.339Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-12-09T15:07:34.339Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-12-09T15:07:34.353Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-12-09T15:07:34.353Zdownloading input: /input/build/work/scripts/test_up.sh
222024-12-09T15:07:34.371Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-12-09T15:07:34.371Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-12-09T15:07:34.387Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-12-09T15:07:34.390Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-12-09T15:07:35.106Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-12-09T15:07:35.106Zstarting task 0: "setup"
282024-12-09T15:07:35.115Z++ uname -s
292024-12-09T15:07:35.118Z+ kern=SunOS
302024-12-09T15:07:35.119Z+ case "$kern" in
312024-12-09T15:07:35.119Z+ groupadd -g 12345 build
322024-12-09T15:07:35.121Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-12-09T15:07:37.127Z+ zfs create -o mountpoint=/work rpool/work
342024-12-09T15:07:37.325Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-12-09T15:07:37.328Z+ home_fs=zfs
362024-12-09T15:07:37.328Z+ [[ zfs == autofs ]]
372024-12-09T15:07:37.328Z+ mkdir -p /home/build
382024-12-09T15:07:37.333Z+ chown build:build /home/build /work
392024-12-09T15:07:39.332Z+ chmod 0700 /home/build /work
402024-12-09T15:07:39.335Zprocess exited: duration 4229 ms, exit code 0
 
412024-12-09T15:07:39.341Zstarting task 1: "authentication"
422024-12-09T15:07:39.365Zprocess exited: duration 24 ms, exit code 0
 
432024-12-09T15:07:39.370Zstarting task 2: "build"
442024-12-09T15:07:39.375Z+ banner cores
452024-12-09T15:07:39.396Z
462024-12-09T15:07:39.396Z #### #### ##### ###### ####
472024-12-09T15:07:39.396Z # # # # # # # #
482024-12-09T15:07:39.396Z # # # # # ##### ####
492024-12-09T15:07:39.396Z # # # ##### # #
502024-12-09T15:07:39.396Z # # # # # # # # #
512024-12-09T15:07:39.396Z #### #### # # ###### ####
522024-12-09T15:07:39.396Z
532024-12-09T15:07:39.406Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-12-09T15:07:39.410Z+ echo 'input bins dir contains:'
552024-12-09T15:07:39.410Zinput bins dir contains:
562024-12-09T15:07:39.410Z+ ls -ltr /input/build/work/bins
572024-12-09T15:07:39.410Ztotal 762799
582024-12-09T15:07:39.410Z-rw-r--r-- 1 root root 109276547 Dec 9 15:07 crucible-downstairs.gz
592024-12-09T15:07:39.410Z-rw-r--r-- 1 root root 105056395 Dec 9 15:07 crucible-hammer.gz
602024-12-09T15:07:39.410Z-rw-r--r-- 1 root root 109488947 Dec 9 15:07 crutest.gz
612024-12-09T15:07:39.410Z-rw-r--r-- 1 root root 66480149 Dec 9 15:07 dsc.gz
622024-12-09T15:07:39.410Z+ banner unpack
632024-12-09T15:07:39.413Z
642024-12-09T15:07:39.413Z # # # # ##### ## #### # #
652024-12-09T15:07:39.413Z # # ## # # # # # # # # #
662024-12-09T15:07:39.413Z # # # # # # # # # # ####
672024-12-09T15:07:39.413Z # # # # # ##### ###### # # #
682024-12-09T15:07:39.413Z # # # ## # # # # # # #
692024-12-09T15:07:39.414Z #### # # # # # #### # #
702024-12-09T15:07:39.414Z
712024-12-09T15:07:39.414Z+ mkdir -p /var/tmp/bins
722024-12-09T15:07:39.414Z+ for t in "$input/bins/"*.gz
732024-12-09T15:07:39.416Z++ basename /input/build/work/bins/crucible-downstairs.gz
742024-12-09T15:07:39.416Z+ b=crucible-downstairs.gz
752024-12-09T15:07:39.416Z+ b=crucible-downstairs
762024-12-09T15:07:39.416Z+ gunzip
772024-12-09T15:07:41.670Z+ chmod +x /var/tmp/bins/crucible-downstairs
782024-12-09T15:07:41.673Z+ for t in "$input/bins/"*.gz
792024-12-09T15:07:41.675Z++ basename /input/build/work/bins/crucible-hammer.gz
802024-12-09T15:07:41.675Z+ b=crucible-hammer.gz
812024-12-09T15:07:41.676Z+ b=crucible-hammer
822024-12-09T15:07:41.676Z+ gunzip
832024-12-09T15:07:43.840Z+ chmod +x /var/tmp/bins/crucible-hammer
842024-12-09T15:07:43.842Z+ for t in "$input/bins/"*.gz
852024-12-09T15:07:43.847Z++ basename /input/build/work/bins/crutest.gz
862024-12-09T15:07:43.847Z+ b=crutest.gz
872024-12-09T15:07:43.847Z+ b=crutest
882024-12-09T15:07:43.847Z+ gunzip
892024-12-09T15:07:46.147Z+ chmod +x /var/tmp/bins/crutest
902024-12-09T15:07:46.150Z+ for t in "$input/bins/"*.gz
912024-12-09T15:07:46.153Z++ basename /input/build/work/bins/dsc.gz
922024-12-09T15:07:46.153Z+ b=dsc.gz
932024-12-09T15:07:46.154Z+ b=dsc
942024-12-09T15:07:46.154Z+ gunzip
952024-12-09T15:07:47.538Z+ chmod +x /var/tmp/bins/dsc
962024-12-09T15:07:47.542Z+ export BINDIR=/var/tmp/bins
972024-12-09T15:07:47.542Z+ BINDIR=/var/tmp/bins
982024-12-09T15:07:47.542Z+ jobpid=1139
992024-12-09T15:07:47.544Z+ echo 'Setup debug logging'
1002024-12-09T15:07:47.544ZSetup debug logging
1012024-12-09T15:07:47.544Z+ mkdir /tmp/debug
1022024-12-09T15:07:47.544Z+ sleep 3600
1032024-12-09T15:07:47.544Z+ psrinfo -v
1042024-12-09T15:07:47.548Z+ df -h
1052024-12-09T15:07:47.575Z+ prstat -d d -mLc 1
1062024-12-09T15:07:47.578Z+ iostat -T d -xn 1
1072024-12-09T15:07:47.578Z+ mpstat -T d 1
1082024-12-09T15:07:47.578Z+ vmstat -T d -p 1
1092024-12-09T15:07:47.580Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1102024-12-09T15:07:47.580Z+ banner test_up_encrypted
1112024-12-09T15:07:47.580Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1122024-12-09T15:07:47.583Z
1132024-12-09T15:07:47.583Z ##### ###### #### ##### # # ##### ###### # #
1142024-12-09T15:07:47.583Z # # # # # # # # # ## #
1152024-12-09T15:07:47.583Z # ##### #### # # # # # ##### # # #
1162024-12-09T15:07:47.583Z # # # # # # ##### # # # #
1172024-12-09T15:07:47.583Z # # # # # # # # # # ##
1182024-12-09T15:07:47.583Z # ###### #### # ####### #### # ####### ###### # #
1192024-12-09T15:07:47.583Z
1202024-12-09T15:07:47.583Z+ ptime -m bash /input/build/work/scripts/test_up.sh -N encrypted
1212024-12-09T15:07:47.592Z/input/build/work
1222024-12-09T15:07:47.624ZTurn off color for downstairs dump
1232024-12-09T15:07:47.656ZUpstairs using key: PWmA3H7iyIcP7Q+LYYKSZNjnsY5+0NfoK9udwM3yRsA=
1242024-12-09T15:07:47.660Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1252024-12-09T15:07:47.660ZCreating 3 downstairs regions
1262024-12-09T15:07:47.979ZStarting 3 downstairs
1272024-12-09T15:07:47.982Zdsc started at PID: 1185
1282024-12-09T15:07:53.165ZDisable automatic restart on all downstairs
1292024-12-09T15:07:53.222Z
1302024-12-09T15:07:53.222ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1312024-12-09T15:07:53.226ZRunning test: span
1322024-12-09T15:07:57.412ZCompleted test: span
1332024-12-09T15:08:02.518ZRunning test: big
1342024-12-09T15:08:06.770ZCompleted test: big
1352024-12-09T15:08:11.778ZRunning test: dep
1362024-12-09T15:09:03.551ZCompleted test: dep
1372024-12-09T15:09:08.559ZRunning test: balloon
1382024-12-09T15:09:48.623ZCompleted test: balloon
1392024-12-09T15:09:53.631ZRunning test: deactivate
1402024-12-09T15:10:48.337ZCompleted test: deactivate
1412024-12-09T15:10:53.345ZRunning hammer
1422024-12-09T15:11:06.648ZRun repair tests
1432024-12-09T15:11:06.652Z/var/tmp/bins/crutest fill -g 31 -q --key PWmA3H7iyIcP7Q+LYYKSZNjnsY5+0NfoK9udwM3yRsA= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1442024-12-09T15:11:06.712Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.096402868Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"task":"crutest"}
1452024-12-09T15:11:06.715Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.096995296Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"task":"crutest"}
1462024-12-09T15:11:06.715Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.098755315Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"task":"crutest"}
1472024-12-09T15:11:06.715Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.09898661Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222}
1482024-12-09T15:11:06.715Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.099063248Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222}
1492024-12-09T15:11:06.715Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.099122019Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222}
1502024-12-09T15:11:06.715Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.099163852Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222}
1512024-12-09T15:11:06.715Z{"msg":"Crucible d066690c-0b03-4776-a063-d33d69150f71 has session id: d5a294ed-308c-4def-8f90-f6c4015b9355","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.09956942Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1522024-12-09T15:11:06.715Z{"msg":"Upstairs opts: Upstairs UUID: d066690c-0b03-4776-a063-d33d69150f71, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.09963563Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1532024-12-09T15:11:06.715Z{"msg":"Crucible stats registered with UUID: d066690c-0b03-4776-a063-d33d69150f71","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.099670124Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1542024-12-09T15:11:06.750Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.134051925Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"task":"crutest"}
1552024-12-09T15:11:06.750Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.134124524Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222}
1562024-12-09T15:11:06.750Z{"msg":"d066690c-0b03-4776-a063-d33d69150f71 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.13414819Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1572024-12-09T15:11:06.750Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.134163188Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"client":"0","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1582024-12-09T15:11:06.750Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.134193083Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"client":"0","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1592024-12-09T15:11:06.750Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.134229278Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"client":"1","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1602024-12-09T15:11:06.754Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.134255323Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"client":"1","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1612024-12-09T15:11:06.754Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.134284769Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"client":"2","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1622024-12-09T15:11:06.754Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.134300256Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"client":"2","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1632024-12-09T15:11:06.754Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.134317674Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"":"io task","client":"2","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1642024-12-09T15:11:06.754Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.134396662Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1652024-12-09T15:11:06.754Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.134460322Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"":"io task","client":"1","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1662024-12-09T15:11:06.754Z{"msg":"ds_connection connected from Ok(127.0.0.1:65495)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.134490877Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"":"io task","client":"2","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1672024-12-09T15:11:06.754Z{"msg":"ds_connection connected from Ok(127.0.0.1:50176)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.134533071Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1682024-12-09T15:11:06.755Z{"msg":"ds_connection connected from Ok(127.0.0.1:57058)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.134561036Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"":"io task","client":"1","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1692024-12-09T15:11:06.755Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.135374381Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"client":"1","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1702024-12-09T15:11:06.755Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.135459938Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"client":"1","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1712024-12-09T15:11:06.755Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.135521098Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"client":"0","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1722024-12-09T15:11:06.755Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.135550094Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"client":"0","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1732024-12-09T15:11:06.755Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.135586728Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"client":"2","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1742024-12-09T15:11:06.755Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.135661137Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"client":"2","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1752024-12-09T15:11:06.755Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.136086651Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"client":"1","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1762024-12-09T15:11:06.755Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.136126455Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"client":"2","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1772024-12-09T15:11:06.755Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.136189415Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"client":"0","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1782024-12-09T15:11:06.755Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.1364832Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"client":"0","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1792024-12-09T15:11:06.755Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.136535242Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"client":"0","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1802024-12-09T15:11:06.755Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.136589464Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1812024-12-09T15:11:06.755Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.13661705Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"client":"1","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1822024-12-09T15:11:06.755Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.136642826Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"client":"1","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1832024-12-09T15:11:06.755Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.13667721Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1842024-12-09T15:11:06.755Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.136711745Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"client":"2","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1852024-12-09T15:11:06.755Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.136749769Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"client":"2","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1862024-12-09T15:11:06.755Z{"msg":"[0]R flush_numbers: [1496, 1496, 1496, 1496, 1496]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.136788663Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":Wait for a query_work_queue command to finish before sending IO
1872024-12-09T15:11:06.755Z1222,"":"downstairs","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1882024-12-09T15:11:06.755Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.136837805Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"":"downstairs","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1892024-12-09T15:11:06.755Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.136880079Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"":"downstairs","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1902024-12-09T15:11:06.755Z{"msg":"[1]R flush_numbers: [1496, 1496, 1496, 1496, 1496]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.136921332Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"":"downstairs","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1912024-12-09T15:11:06.755Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.136960197Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"":"downstairs","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1922024-12-09T15:11:06.755Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
1932024-12-09T15:11:06.755Z,"time":"2024-12-09T15:11:06.137000191Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"":"downstairs","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1942024-12-09T15:11:06.755Z{"msg":"[2]R flush_numbers: [1496, 1496, 1496, 1496, 1496]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.137032755Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":Fill test
1952024-12-09T15:11:06.755Z1222,"":"downstairs","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1962024-12-09T15:11:06.755Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.137071249Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"":"downstairs","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1972024-12-09T15:11:06.755Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.137100015Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"":"downstairs","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1982024-12-09T15:11:06.755Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.137115493Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"":"downstairs","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
1992024-12-09T15:11:06.755Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.137137129Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"":"downstairs","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
2002024-12-09T15:11:06.755Z{"msg":"Next flush: 1497","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.137151007Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"":"downstairs","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
2012024-12-09T15:11:06.755Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.137164795Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"":"downstairs","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
2022024-12-09T15:11:06.755Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.137185522Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
2032024-12-09T15:11:06.755Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.13719934Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"client":"0","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
2042024-12-09T15:11:06.755Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.137220517Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"client":"1","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
2052024-12-09T15:11:06.755Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.137235364Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"client":"2","":"downstairs","session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
2062024-12-09T15:11:06.755Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.137256531Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
2072024-12-09T15:11:06.756Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.137269689Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
2082024-12-09T15:11:06.756Z{"msg":"d066690c-0b03-4776-a063-d33d69150f71 is now active with session: d5a294ed-308c-4def-8f90-f6c4015b9355","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.137301384Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
2092024-12-09T15:11:06.756Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.137336029Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222,"session_id":"d5a294ed-308c-4def-8f90-f6c4015b9355"}
2102024-12-09T15:11:06.756Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.137369394Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222}
2112024-12-09T15:11:06.791ZRead and Verify all blocks (0..50 range:false)
2122024-12-09T15:11:06.830ZWrote out file "/var/tmp/test_up-build/verify_file"
2132024-12-09T15:11:06.830ZCLIENT: Tests done. All submitted work has been ACK'd
2142024-12-09T15:11:06.834Z----------------------------------------------------------------
2152024-12-09T15:11:06.834Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2162024-12-09T15:11:06.834ZStates: Active Active Active
2172024-12-09T15:11:06.834ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2182024-12-09T15:11:06.834Z 1002 Acked Read 50 Done Done Done false
2192024-12-09T15:11:06.834Z STATES DS:0 DS:1 DS:2 TOTAL
2202024-12-09T15:11:06.834Z Sent 0 0 0 0
2212024-12-09T15:11:06.834Z Done 1 1 1 3
2222024-12-09T15:11:06.834Z Skipped 0 0 0 0
2232024-12-09T15:11:06.834Z Error 0 0 0 0
2242024-12-09T15:11:06.834ZLast Flush: 1001 1001 1001
2252024-12-09T15:11:06.834ZDownstairs last five completed: 1001 1000
2262024-12-09T15:11:06.834ZUpstairs last five completed: 1002 1001 1000
2272024-12-09T15:11:06.834ZCLIENT: Up:0 ds:1 act:3
2282024-12-09T15:11:10.923Z----------------------------------------------------------------
2292024-12-09T15:11:10.923Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2302024-12-09T15:11:10.923ZDownstairs last five completed: 1003 1002 1001 1000
2312024-12-09T15:11:10.923ZUpstairs last five completed: 1003 1002 1001 1000
2322024-12-09T15:11:10.923ZCLIENT: Up:0 ds:0 act:3
2332024-12-09T15:11:10.923ZCLIENT: All crucible jobs finished, exiting program
2342024-12-09T15:11:10.926Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:10.216411466Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1222}
2352024-12-09T15:11:10.926ZRepair setup passed
2362024-12-09T15:11:10.926ZCopy the region for /var/tmp/test_up-build/8830
2372024-12-09T15:11:10.926Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2382024-12-09T15:11:10.926Z/var/tmp/bins/crutest fill -g 32 -q --key PWmA3H7iyIcP7Q+LYYKSZNjnsY5+0NfoK9udwM3yRsA= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2392024-12-09T15:11:10.926Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.294781667Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"task":"crutest"}
2402024-12-09T15:11:10.926Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.29534812Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"task":"crutest"}
2412024-12-09T15:11:10.926Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.297023332Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"task":"crutest"}
2422024-12-09T15:11:10.926Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.297258075Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228}
2432024-12-09T15:11:10.926Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.297299989Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228}
2442024-12-09T15:11:10.926Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.297323815Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228}
2452024-12-09T15:11:10.926Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.297340963Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228}
2462024-12-09T15:11:10.926Z{"msg":"Crucible 16787630-d23e-4517-9233-3bc61917cc07 has session id: 8e6036cc-d670-4505-bab5-cf509a89490e","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.297825358Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2472024-12-09T15:11:10.926Z{"msg":"Upstairs opts: Upstairs UUID: 16787630-d23e-4517-9233-3bc61917cc07, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.297866712Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2482024-12-09T15:11:10.926Z{"msg":"Crucible stats registered with UUID: 16787630-d23e-4517-9233-3bc61917cc07","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.297883459Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2492024-12-09T15:11:10.949Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.332249342Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"task":"crutest"}
2502024-12-09T15:11:10.949Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.332312223Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228}
2512024-12-09T15:11:10.952Z{"msg":"16787630-d23e-4517-9233-3bc61917cc07 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.33239125Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2522024-12-09T15:11:10.952Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.332434214Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"client":"0","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2532024-12-09T15:11:10.952Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.332507672Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"client":"0","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2542024-12-09T15:11:10.952Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.332533898Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"client":"1","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2552024-12-09T15:11:10.952Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.332554005Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"client":"1","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2562024-12-09T15:11:10.952Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.332573052Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"client":"2","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2572024-12-09T15:11:10.952Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.332600148Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"client":"2","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2582024-12-09T15:11:10.952Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.332649051Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"":"io task","client":"2","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2592024-12-09T15:11:10.952Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.332700093Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"":"io task","client":"0","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2602024-12-09T15:11:10.952Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.332752195Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"":"io task","client":"1","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2612024-12-09T15:11:10.952Z{"msg":"ds_connection connected from Ok(127.0.0.1:62992)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.332788139Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"":"io task","client":"2","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2622024-12-09T15:11:10.952Z{"msg":"ds_connection connected from Ok(127.0.0.1:62532)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.332819444Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"":"io task","client":"1","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2632024-12-09T15:11:10.952Z{"msg":"ds_connection connected from Ok(127.0.0.1:51212)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.33285112Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"":"io task","client":"0","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2642024-12-09T15:11:10.952Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.333577337Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"client":"1","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2652024-12-09T15:11:10.952Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.333617081Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"client":"1","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2662024-12-09T15:11:10.952Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.333634629Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"client":"0","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2672024-12-09T15:11:10.952Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.333673053Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"client":"0","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2682024-12-09T15:11:10.952Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.333718316Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"client":"2","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2692024-12-09T15:11:10.952Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.333744272Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"client":"2","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2702024-12-09T15:11:10.952Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.334249554Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"client":"0","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2712024-12-09T15:11:10.952Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.334382903Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"client":"1","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2722024-12-09T15:11:10.952Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.33447098Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"client":"2","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2732024-12-09T15:11:10.952Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.334658571Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"client":"0","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2742024-12-09T15:11:10.952Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.334701894Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"client":"0","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2752024-12-09T15:11:10.952Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.33472893Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2762024-12-09T15:11:10.952Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.334762375Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"client":"2","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2772024-12-09T15:11:10.955Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.334789051Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"client":"2","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2782024-12-09T15:11:10.955Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.334818806Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2792024-12-09T15:11:10.955Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.334842322Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"client":"1","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2802024-12-09T15:11:10.955Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.334876387Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"client":"1","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2812024-12-09T15:11:10.955Z{"msg":"[0]R flush_numbers: [1497, 1497, 1497, 1497, 1497]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.334902163Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"":"downstairs","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2822024-12-09T15:11:10.955Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.334931639Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"":"downstairs","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2832024-12-09T15:11:10.955Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.334957235Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"":"downstairs","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2842024-12-09T15:11:10.955Z{"msg":"[1]R flush_numbers: [1497, 1497, 1497, 1497, 1497]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.334982691Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"":"downstairs","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2852024-12-09T15:11:10.955Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.335007997Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"":"downstairs","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2862024-12-09T15:11:10.955ZWait for a query_work_queue command to finish before sending IO
2872024-12-09T15:11:10.955Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.335043582Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"":"downstairs","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2882024-12-09T15:11:10.955Z{"msg":"[2]R flush_numbers: [1497, 1497, 1497, 1497, 1497]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.335079446Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"":"downstairs","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2892024-12-09T15:11:10.955Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.335098633Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"":"downstairs","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2902024-12-09T15:11:10.955Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.335139617Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"":"downstairs","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2912024-12-09T15:11:10.955Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.335166253Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"":"downstairs","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2922024-12-09T15:11:10.955Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.335198798Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"":"downstairs","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2932024-12-09T15:11:10.955Z{"msg":"Next flush: 1498","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.335218145Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"":"downstairs","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2942024-12-09T15:11:10.955Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.33525163Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"":"downstairs","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2952024-12-09T15:11:10.955Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.335281615Z","Disk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2962024-12-09T15:11:10.956Zhostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2972024-12-09T15:11:10.956Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.335319419Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"client":"0","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2982024-12-09T15:11:10.956Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.335352344Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"client":"1","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
2992024-12-09T15:11:10.956Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.335383659Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"client":"2","":"downstairs","session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
3002024-12-09T15:11:10.956Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.335403836Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
3012024-12-09T15:11:10.956Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.335433451Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
3022024-12-09T15:11:10.956Z{"msg":"16787630-d23e-4517-9233-3bc61917cc07 is now active with session: 8e6036cc-d670-4505-bab5-cf509a89490e","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.335452268Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
3032024-12-09T15:11:10.956Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30Loading write count information from file "/var/tmp/test_up-build/verify_file"
3042024-12-09T15:11:10.956Z,"time":"2024-12-09T15:11:10.335480784Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228}
3052024-12-09T15:11:10.956Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:10.335502411Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228,"session_id":"8e6036cc-d670-4505-bab5-cf509a89490e"}
3062024-12-09T15:11:10.956ZFill test
3072024-12-09T15:11:10.989ZRead and Verify all blocks (0..50 range:false)
3082024-12-09T15:11:11.027ZWrote out file "/var/tmp/test_up-build/verify_file"
3092024-12-09T15:11:11.027ZCLIENT: Tests done. All submitted work has been ACK'd
3102024-12-09T15:11:11.030Z----------------------------------------------------------------
3112024-12-09T15:11:11.030Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
3122024-12-09T15:11:11.030ZStates: Active Active Active
3132024-12-09T15:11:11.030ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
3142024-12-09T15:11:11.030Z 1002 Acked Read 50 Done Done Done false
3152024-12-09T15:11:11.030Z STATES DS:0 DS:1 DS:2 TOTAL
3162024-12-09T15:11:11.030Z Sent 0 0 0 0
3172024-12-09T15:11:11.030Z Done 1 1 1 3
3182024-12-09T15:11:11.030Z Skipped 0 0 0 0
3192024-12-09T15:11:11.030Z Error 0 0 0 0
3202024-12-09T15:11:11.030ZLast Flush: 1001 1001 1001
3212024-12-09T15:11:11.030ZDownstairs last five completed: 1001 1000
3222024-12-09T15:11:11.030ZUpstairs last five completed: 1002 1001 1000
3232024-12-09T15:11:11.030ZCLIENT: Up:0 ds:1 act:3
3242024-12-09T15:11:15.029Z----------------------------------------------------------------
3252024-12-09T15:11:15.029Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
3262024-12-09T15:11:15.030ZDownstairs last five completed: 1003 1002 1001 1000
3272024-12-09T15:11:15.030ZUpstairs last five completed: 1003 1002 1001 1000
3282024-12-09T15:11:15.030ZCLIENT: Up:0 ds:0 act:3
3292024-12-09T15:11:15.030ZCLIENT: All crucible jobs finished, exiting program
3302024-12-09T15:11:15.032Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:14.412358571Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1228}
3312024-12-09T15:11:15.036ZRepair part 1 passed
3322024-12-09T15:11:15.039Z
3332024-12-09T15:11:15.039ZKill the current downstairs
3342024-12-09T15:11:15.213ZDownstairs 2 stopped
3352024-12-09T15:11:15.213Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
3362024-12-09T15:11:15.216ZNow put back the original so we have a mismatch
3372024-12-09T15:11:15.216Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
3382024-12-09T15:11:15.218ZRestart downstairs with old directory
3392024-12-09T15:11:15.277Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
3402024-12-09T15:11:15.293ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3412024-12-09T15:11:15.296Z 0 000-009 32 32 31 1498 1498 1497 F F F <---
3422024-12-09T15:11:15.296Z 1 010-019 32 32 31 1498 1498 1497 F F F <---
3432024-12-09T15:11:15.296Z 2 020-029 32 32 31 1498 1498 1497 F F F <---
3442024-12-09T15:11:15.296Z 3 030-039 32 32 31 1498 1498 1497 F F F <---
3452024-12-09T15:11:15.296Z 4 040-049 32 32 31 1498 1498 1497 F F F <---
3462024-12-09T15:11:15.296ZMax gen: 32, Max flush: 1498
3472024-12-09T15:11:15.296ZError: Difference in extent metadata found!
3482024-12-09T15:11:15.298Zdump test found error as expected
3492024-12-09T15:11:15.298Z
3502024-12-09T15:11:15.298Z
3512024-12-09T15:11:15.298Z/var/tmp/bins/crutest verify --range -g 33 -q --key PWmA3H7iyIcP7Q+LYYKSZNjnsY5+0NfoK9udwM3yRsA= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
3522024-12-09T15:11:15.357Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.739082283Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"task":"crutest"}
3532024-12-09T15:11:15.359Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.739664213Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"task":"crutest"}
3542024-12-09T15:11:15.360Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.741588956Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"task":"crutest"}
3552024-12-09T15:11:15.360Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.74188692Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236}
3562024-12-09T15:11:15.360Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.741928294Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236}
3572024-12-09T15:11:15.360Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.741961399Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236}
3582024-12-09T15:11:15.360Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.741986295Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236}
3592024-12-09T15:11:15.363Z{"msg":"Crucible 19bc867a-e530-4af5-b117-c7ee5cc283ff has session id: e480e8d1-42db-4f23-aa6a-feaace785bcb","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.74266454Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3602024-12-09T15:11:15.363Z{"msg":"Upstairs opts: Upstairs UUID: 19bc867a-e530-4af5-b117-c7ee5cc283ff, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.742728961Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3612024-12-09T15:11:15.363Z{"msg":"Crucible stats registered with UUID: 19bc867a-e530-4af5-b117-c7ee5cc283ff","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.742746988Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3622024-12-09T15:11:15.396Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.778045567Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"task":"crutest"}
3632024-12-09T15:11:15.396Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.778121815Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236}
3642024-12-09T15:11:15.396Z{"msg":"19bc867a-e530-4af5-b117-c7ee5cc283ff active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.77815271Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3652024-12-09T15:11:15.396Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.778172727Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"client":"0","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3662024-12-09T15:11:15.399Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.778216021Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"client":"0","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3672024-12-09T15:11:15.399Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.778264123Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"client":"1","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3682024-12-09T15:11:15.399Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.778317415Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"client":"1","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3692024-12-09T15:11:15.400Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.778381695Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"client":"2","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3702024-12-09T15:11:15.400Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.778443156Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"client":"2","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3712024-12-09T15:11:15.400Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.77848224Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"":"io task","client":"2","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3722024-12-09T15:11:15.400Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.778519644Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"":"io task","client":"0","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3732024-12-09T15:11:15.400Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.778810869Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"":"io task","client":"1","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3742024-12-09T15:11:15.400Z{"msg":"ds_connection connected from Ok(127.0.0.1:51774)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.778882208Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"":"io task","client":"0","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3752024-12-09T15:11:15.400Z{"msg":"ds_connection connected from Ok(127.0.0.1:60070)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.778925361Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"":"io task","client":"1","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3762024-12-09T15:11:15.400Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:14.778962246Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"":"io task","client":"2","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3772024-12-09T15:11:15.400Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:14.779016897Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"":"io task","client":"2","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3782024-12-09T15:11:15.400Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:14.779076748Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3792024-12-09T15:11:15.400Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.779108923Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"":"io task","client":"2","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3802024-12-09T15:11:15.400Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.779341887Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"client":"0","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3812024-12-09T15:11:15.400Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.779379851Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"client":"0","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3822024-12-09T15:11:15.400Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.779498903Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"client":"1","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3832024-12-09T15:11:15.400Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.779527658Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"client":"1","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3842024-12-09T15:11:15.400Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.779868126Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"client":"0","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3852024-12-09T15:11:15.400Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.780015483Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"client":"1","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3862024-12-09T15:11:15.400Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.780257126Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"client":"0","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3872024-12-09T15:11:15.400Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.780291941Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"client":"0","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3882024-12-09T15:11:15.400Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.780307738Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3892024-12-09T15:11:15.400Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.780320366Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"client":"1","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3902024-12-09T15:11:15.400Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.780333964Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"client":"1","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3912024-12-09T15:11:15.400Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:14.780355221Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3922024-12-09T15:11:25.399Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.779655907Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"":"io task","client":"2","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3932024-12-09T15:11:25.403Z{"msg":"ds_connection connected from Ok(127.0.0.1:35931)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.779819772Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"":"io task","client":"2","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3942024-12-09T15:11:25.403Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.781002399Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"client":"2","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3952024-12-09T15:11:25.403Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.781050662Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"client":"2","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3962024-12-09T15:11:25.403Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.781585649Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"client":"2","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3972024-12-09T15:11:25.403Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782007654Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"client":"2","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3982024-12-09T15:11:25.403Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782045968Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"client":"2","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
3992024-12-09T15:11:25.403Z{"msg":"[0]R flush_numbers: [1498, 1498, 1498, 1498, 1498]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782065355Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4002024-12-09T15:11:25.403Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782088382Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4012024-12-09T15:11:25.403Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.7821035Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4022024-12-09T15:11:25.403Z{"msg":"[1]R flush_numbers: [1498, 1498, 1498, 1498, 1498]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782125466Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4032024-12-09T15:11:25.403Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782140904Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4042024-12-09T15:11:25.403Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782161611Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4052024-12-09T15:11:25.403Z{"msg":"[2]R flush_numbers: [1497, 1497, 1497, 1497, 1497]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782175668Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4062024-12-09T15:11:25.403Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782196395Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4072024-12-09T15:11:25.403Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782210823Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4082024-12-09T15:11:25.403Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782224971Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4092024-12-09T15:11:25.403Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782238629Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4102024-12-09T15:11:25.403Z{"msg":"Next flush: 1499","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782252157Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4112024-12-09T15:11:25.403Z{"msg":"Extent 0 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782266364Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4122024-12-09T15:11:25.403Z{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782280362Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4132024-12-09T15:11:25.403Z{"msg":"extent:0 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782302299Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4142024-12-09T15:11:25.403Z{"msg":"extent:0 flush: 1498 1498 1497 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782317836Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4152024-12-09T15:11:25.403Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782339223Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4162024-12-09T15:11:25.403Z{"msg":"extent:0 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782354231Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4172024-12-09T15:11:25.403Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782375747Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4182024-12-09T15:11:25.403Z{"msg":"find dest for source 0 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782390855Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4192024-12-09T15:11:25.403Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782412782Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4202024-12-09T15:11:25.403Z{"msg":"Extent 1 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782438818Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4212024-12-09T15:11:25.403Z{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782463174Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4222024-12-09T15:11:25.404Z{"msg":"extent:1 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782478572Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4232024-12-09T15:11:25.404Z{"msg":"extent:1 flush: 1498 1498 1497 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782500938Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4242024-12-09T15:11:25.404Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782516396Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4252024-12-09T15:11:25.404Z{"msg":"extent:1 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782532003Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4262024-12-09T15:11:25.404Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782547221Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4272024-12-09T15:11:25.404Z{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782568868Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4282024-12-09T15:11:25.404Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782583826Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4292024-12-09T15:11:25.404Z{"msg":"Extent 2 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782605532Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4302024-12-09T15:11:25.404Z{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.7826201Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4312024-12-09T15:11:25.404Z{"msg":"extent:2 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782641237Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4322024-12-09T15:11:25.404Z{"msg":"extent:2 flush: 1498 1498 1497 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782656654Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4332024-12-09T15:11:25.404Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782678291Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4342024-12-09T15:11:25.404Z{"msg":"extent:2 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782704537Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4352024-12-09T15:11:25.408Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782734012Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4362024-12-09T15:11:25.408Z{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782759448Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4372024-12-09T15:11:25.408Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782790903Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4382024-12-09T15:11:25.408Z{"msg":"Extent 3 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782808821Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4392024-12-09T15:11:25.408Z{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782839296Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4402024-12-09T15:11:25.408Z{"msg":"extent:3 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782862922Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4412024-12-09T15:11:25.408Z{"msg":"extent:3 flush: 1498 1498 1497 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782886739Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4422024-12-09T15:11:25.408Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782925263Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4432024-12-09T15:11:25.408Z{"msg":"extent:3 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782951049Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4442024-12-09T15:11:25.408Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.782992403Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4452024-12-09T15:11:25.408Z{"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.783038695Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4462024-12-09T15:11:25.408Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.78307691Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4472024-12-09T15:11:25.408Z{"msg":"Extent 4 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.783113244Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4482024-12-09T15:11:25.409Z{"msg":"First source client ID for extent 4","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.783129681Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4492024-12-09T15:11:25.409Z{"msg":"extent:4 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.783144369Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4502024-12-09T15:11:25.409Z{"msg":"extent:4 flush: 1498 1498 1497 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.783167405Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4512024-12-09T15:11:25.409Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.783182833Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4522024-12-09T15:11:25.409Z{"msg":"extent:4 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.783198671Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4532024-12-09T15:11:25.409Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.783213618Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4542024-12-09T15:11:25.409Z{"msg":"find dest for source 0 for extent at index 4","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.783228316Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4552024-12-09T15:11:25.409Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.783250033Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4562024-12-09T15:11:25.409Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.78326551Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"client":"0","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4572024-12-09T15:11:25.409Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.783286527Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"client":"1","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4582024-12-09T15:11:25.409Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.783300865Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"client":"2","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4592024-12-09T15:11:25.409Z{"msg":"Full repair list: {ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.783325971Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4602024-12-09T15:11:25.409Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 1499, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 1499, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 1499, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 1499, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 1499, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.783439274Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4612024-12-09T15:11:25.409Z{"msg":"starting reconciliation dc0081c5-5c24-4015-aee1-2ed39f6b2621: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.783483257Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4622024-12-09T15:11:25.409Z{"msg":"no Nexus client from DNS, aborting start notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:11:24.783511352Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"reconcile":"dc0081c5-5c24-4015-aee1-2ed39f6b2621","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4632024-12-09T15:11:25.409Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:11:24.783548127Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"reconcile":"dc0081c5-5c24-4015-aee1-2ed39f6b2621","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4642024-12-09T15:11:25.409Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:11:24.783577802Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"reconcile":"dc0081c5-5c24-4015-aee1-2ed39f6b2621","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4652024-12-09T15:11:25.409Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.783718451Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"client":"2","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4662024-12-09T15:11:25.409Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:11:24.783777282Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"reconcile":"dc0081c5-5c24-4015-aee1-2ed39f6b2621","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4672024-12-09T15:11:25.414Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:11:24.788481237Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"reconcile":"dc0081c5-5c24-4015-aee1-2ed39f6b2621","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4682024-12-09T15:11:25.414Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:11:24.788936976Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"reconcile":"dc0081c5-5c24-4015-aee1-2ed39f6b2621","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4692024-12-09T15:11:25.414Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:11:24.789283303Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"reconcile":"dc0081c5-5c24-4015-aee1-2ed39f6b2621","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4702024-12-09T15:11:25.414Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.789902927Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"client":"2","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4712024-12-09T15:11:25.414Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:11:24.789958159Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"reconcile":"dc0081c5-5c24-4015-aee1-2ed39f6b2621","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4722024-12-09T15:11:25.414Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:11:24.792877899Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"reconcile":"dc0081c5-5c24-4015-aee1-2ed39f6b2621","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4732024-12-09T15:11:25.414Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:11:24.79332618Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"reconcile":"dc0081c5-5c24-4015-aee1-2ed39f6b2621","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4742024-12-09T15:11:25.417Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:11:24.793636482Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"reconcile":"dc0081c5-5c24-4015-aee1-2ed39f6b2621","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4752024-12-09T15:11:25.417Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.79403727Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"client":"2","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4762024-12-09T15:11:25.417Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:11:24.794080043Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"reconcile":"dc0081c5-5c24-4015-aee1-2ed39f6b2621","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4772024-12-09T15:11:25.417Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:11:24.796963759Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"reconcile":"dc0081c5-5c24-4015-aee1-2ed39f6b2621","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4782024-12-09T15:11:25.417Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:11:24.797435196Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"reconcile":"dc0081c5-5c24-4015-aee1-2ed39f6b2621","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4792024-12-09T15:11:25.417Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:11:24.797678369Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"reconcile":"dc0081c5-5c24-4015-aee1-2ed39f6b2621","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4802024-12-09T15:11:25.420Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.79806103Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"client":"2","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4812024-12-09T15:11:25.420Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:11:24.798104203Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"reconcile":"dc0081c5-5c24-4015-aee1-2ed39f6b2621","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4822024-12-09T15:11:25.422Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:11:24.800876046Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"reconcile":"dc0081c5-5c24-4015-aee1-2ed39f6b2621","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4832024-12-09T15:11:25.422Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:11:24.801242479Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"reconcile":"dc0081c5-5c24-4015-aee1-2ed39f6b2621","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4842024-12-09T15:11:25.422Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:11:24.8014956Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"reconcile":"dc0081c5-5c24-4015-aee1-2ed39f6b2621","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4852024-12-09T15:11:25.422Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.801793195Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"client":"2","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4862024-12-09T15:11:25.422Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:11:24.80182552Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"reconcile":"dc0081c5-5c24-4015-aee1-2ed39f6b2621","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4872024-12-09T15:11:25.426Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:11:24.804736561Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"reconcile":"dc0081c5-5c24-4015-aee1-2ed39f6b2621","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4882024-12-09T15:11:25.426Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.805145468Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4892024-12-09T15:11:25.426Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.805179303Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"client":"0","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4902024-12-09T15:11:25.426Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.80519684Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"client":"1","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4912024-12-09T15:11:25.426Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.805211008Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"client":"2","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4922024-12-09T15:11:25.426Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.805234684Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4932024-12-09T15:11:25.426Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.805248192Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4942024-12-09T15:11:25.426Z{"msg":"19bc867a-e530-4af5-b117-c7ee5cc283ff is now active with session: e480e8d1-42db-4f23-aa6a-feaace785bcb","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.80526207Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4952024-12-09T15:11:25.426Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.805283097Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236}
4962024-12-09T15:11:25.426ZWait for a query_work_queue command to finish before sending IO
4972024-12-09T15:11:25.426Z{"msg":"no Nexus client from DNS, aborting finish notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:11:24.805351206Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236,"reconcile":"dc0081c5-5c24-4015-aee1-2ed39f6b2621","":"downstairs","":"downstairs","session_id":"e480e8d1-42db-4f23-aa6a-feaace785bcb"}
4982024-12-09T15:11:25.426ZDisk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
4992024-12-09T15:11:25.426ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
5002024-12-09T15:11:25.426ZRead and Verify all blocks (0..50 range:true)
5012024-12-09T15:11:25.464ZWrote out file "/var/tmp/test_up-build/verify_file"
5022024-12-09T15:11:25.464ZVerify test completed
5032024-12-09T15:11:25.467ZWrote out file "/var/tmp/test_up-build/verify_file"
5042024-12-09T15:11:25.467ZCLIENT: Tests done. All submitted work has been ACK'd
5052024-12-09T15:11:25.467Z----------------------------------------------------------------
5062024-12-09T15:11:25.467Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
5072024-12-09T15:11:25.467ZStates: Active Active Active
5082024-12-09T15:11:25.467ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
5092024-12-09T15:11:25.467Z 1000 Acked Read 50 Done Done Done false
5102024-12-09T15:11:25.467Z STATES DS:0 DS:1 DS:2 TOTAL
5112024-12-09T15:11:25.467Z Sent 0 0 0 0
5122024-12-09T15:11:25.467Z Done 1 1 1 3
5132024-12-09T15:11:25.467Z Skipped 0 0 0 0
5142024-12-09T15:11:25.467Z Error 0 0 0 0
5152024-12-09T15:11:25.467ZLast Flush: 0 0 0
5162024-12-09T15:11:25.467ZDownstairs last five completed:
5172024-12-09T15:11:25.467ZUpstairs last five completed: 1000
5182024-12-09T15:11:25.467ZCLIENT: Up:0 ds:1 act:3
5192024-12-09T15:11:29.467Z----------------------------------------------------------------
5202024-12-09T15:11:29.467Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
5212024-12-09T15:11:29.467ZDownstairs last five completed: 1001 1000
5222024-12-09T15:11:29.467ZUpstairs last five completed: 1001 1000
5232024-12-09T15:11:29.467ZCLIENT: Up:0 ds:0 act:3
5242024-12-09T15:11:29.467ZCLIENT: All crucible jobs finished, exiting program
5252024-12-09T15:11:29.470Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:28.847392352Z","hostname":"w-01JENZAAS3HDWWSJJ20Q5Q6PP4","pid":1236}
5262024-12-09T15:11:29.474ZRepair part 2 passed
5272024-12-09T15:11:29.477Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
5282024-12-09T15:11:29.492ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
5292024-12-09T15:11:29.492Z 0 000-009 32 32 32 1498 1498 1498 F F F
5302024-12-09T15:11:29.492Z 1 010-019 32 32 32 1498 1498 1498 F F F
5312024-12-09T15:11:29.492Z 2 020-029 32 32 32 1498 1498 1498 F F F
5322024-12-09T15:11:29.492Z 3 030-039 32 32 32 1498 1498 1498 F F F
5332024-12-09T15:11:29.492Z 4 040-049 32 32 32 1498 1498 1498 F F F
5342024-12-09T15:11:29.492ZMax gen: 32, Max flush: 1498
5352024-12-09T15:11:29.497Zdump test passed
5362024-12-09T15:11:29.500Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
5372024-12-09T15:11:29.514Z Extent 1
5382024-12-09T15:11:29.514ZGEN 32 32 32
5392024-12-09T15:11:29.514ZFLUSH_ID 1498 1498 1498
5402024-12-09T15:11:29.514ZDIRTY
5412024-12-09T15:11:29.515Z
5422024-12-09T15:11:29.515ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
5432024-12-09T15:11:29.518Z010 A A A A A A
5442024-12-09T15:11:29.518Z011 A A A A A A
5452024-12-09T15:11:29.521Z012 A A A A A A
5462024-12-09T15:11:29.521Z013 A A A A A A
5472024-12-09T15:11:29.524Z014 A A A A A A
5482024-12-09T15:11:29.524Z015 A A A A A A
5492024-12-09T15:11:29.524Z016 A A A A A A
5502024-12-09T15:11:29.526Z017 A A A A A A
5512024-12-09T15:11:29.526Z018 A A A A A A
5522024-12-09T15:11:29.529Z019 A A A A A A
5532024-12-09T15:11:29.533Zdump extent test passed
5542024-12-09T15:11:29.535Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
5552024-12-09T15:11:29.551ZExtent 2 Block in extent 0 Actual block 20
5562024-12-09T15:11:29.551Z
5572024-12-09T15:11:29.553Z DATA SHA256 VER
5582024-12-09T15:11:29.553Z------ ---------------------------------------------------------------- ---
5592024-12-09T15:11:29.553Z 0 d076969eb501c8c70881ab6a8ac0826de4f01a4ca8e271809c7123bd84477e17 A
5602024-12-09T15:11:29.553Z 1 d076969eb501c8c70881ab6a8ac0826de4f01a4ca8e271809c7123bd84477e17 A
5612024-12-09T15:11:29.553Z 2 d076969eb501c8c70881ab6a8ac0826de4f01a4ca8e271809c7123bd84477e17 A
5622024-12-09T15:11:29.553Z
5632024-12-09T15:11:29.553ZNONCES 0 1 2 DIFF
5642024-12-09T15:11:29.553Z------ ------------------------ ------------------------ ------------------------ -----
5652024-12-09T15:11:29.553Z 0 ee1b5b72ca7bcafc7dc3985f ee1b5b72ca7bcafc7dc3985f ee1b5b72ca7bcafc7dc3985f
5662024-12-09T15:11:29.554Z
5672024-12-09T15:11:29.554Z TAGS 0 1 2 DIFF
5682024-12-09T15:11:29.554Z------ -------------------------------- -------------------------------- -------------------------------- -----
5692024-12-09T15:11:29.554Z 0 dc726cf6f30d7ab8f1d089093c9dae4c dc726cf6f30d7ab8f1d089093c9dae4c dc726cf6f30d7ab8f1d089093c9dae4c
5702024-12-09T15:11:29.554Z
5712024-12-09T15:11:29.554ZHASHES 0 1 2 DIFF
5722024-12-09T15:11:29.554Z------ ---------------- ---------------- ---------------- -----
5732024-12-09T15:11:29.554Z
5742024-12-09T15:11:29.559Zdump block test passed
5752024-12-09T15:11:29.561ZInitial upstairs tests have completed, stopping all downstairs
5762024-12-09T15:11:34.626ZCreating 4 larger downstairs regions
5772024-12-09T15:11:34.743ZStarting 4 downstairs
5782024-12-09T15:11:39.751Zdsc restarted at PID: 1249
5792024-12-09T15:11:39.986ZNow do the replace-reconcile test
5802024-12-09T15:11:40.044ZUsing 8840 for the replacement port
5812024-12-09T15:14:14.899ZCompleted test: replace-reconcile
5822024-12-09T15:14:14.899ZNow do the replace-before-active test
5832024-12-09T15:17:21.363ZCompleted test: replace-before-active
5842024-12-09T15:17:21.363ZAll tests have completed, stopping all downstairs
5852024-12-09T15:17:21.423Z
5862024-12-09T15:17:21.423ZAll Tests have passed
5872024-12-09T15:17:21.426Z9:33 Test duration
5882024-12-09T15:17:21.426Z
5892024-12-09T15:17:21.426Zreal 9:33.739172369
5902024-12-09T15:17:21.426Zuser 6:20.064010587
5912024-12-09T15:17:21.426Zsys 12.597936601
5922024-12-09T15:17:21.426Ztrap 0.213024825
5932024-12-09T15:17:21.426Ztflt 0.024032446
5942024-12-09T15:17:21.426Zdflt 0.066028147
5952024-12-09T15:17:21.426Zkflt 0.001347303
5962024-12-09T15:17:21.426Zlock 8:22:18.697037303
5972024-12-09T15:17:21.426Zslp 33:14.338948076
5982024-12-09T15:17:21.426Zlat 23.556722807
5992024-12-09T15:17:21.426Zstop 0.127314304
6002024-12-09T15:17:21.427Z+ echo 'test-up-encrypted ends'
6012024-12-09T15:17:21.427Ztest-up-encrypted ends
6022024-12-09T15:17:26.430Zprocess exited: duration 581953 ms, exit code 0
6032024-12-09T15:17:26.430Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
6042024-12-09T15:18:26.473Zexec warning: : stdout descriptor may be held open by a background process; giving up!
6052024-12-09T15:18:26.473Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
6062024-12-09T15:18:26.484Zfound 14 output files
6072024-12-09T15:18:26.484Zuploading: /tmp/test_up-build/dsc-out.txt (37484 bytes)
6082024-12-09T15:18:27.495Zuploaded: /tmp/test_up-build/dsc-out.txt
6092024-12-09T15:18:27.495Zuploading: /tmp/test_up-build/test_up_out.txt (10603432 bytes)
6102024-12-09T15:18:28.636Zuploaded: /tmp/test_up-build/test_up_out.txt
6112024-12-09T15:18:28.636Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (2499 bytes)
6122024-12-09T15:18:29.644Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
6132024-12-09T15:18:29.646Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (144657 bytes)
6142024-12-09T15:18:30.655Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
6152024-12-09T15:18:30.658Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (251114 bytes)
6162024-12-09T15:18:31.669Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
6172024-12-09T15:18:31.672Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (139419 bytes)
6182024-12-09T15:18:32.683Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
6192024-12-09T15:18:32.689Zuploading: /tmp/debug/df.txt (1270 bytes)
6202024-12-09T15:18:33.694Zuploaded: /tmp/debug/df.txt
6212024-12-09T15:18:33.697Zuploading: /tmp/debug/dtrace.txt (471434 bytes)
6222024-12-09T15:18:34.710Zuploaded: /tmp/debug/dtrace.txt
6232024-12-09T15:18:34.712Zuploading: /tmp/debug/iostat.txt (192403 bytes)
6242024-12-09T15:18:34.718Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 192403 -> 195112
6252024-12-09T15:18:35.725Zuploaded: /tmp/debug/iostat.txt
6262024-12-09T15:18:35.725Zuploading: /tmp/debug/mpstat.txt (466475 bytes)
6272024-12-09T15:18:35.736Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 466475 -> 473775
6282024-12-09T15:18:36.742Zuploaded: /tmp/debug/mpstat.txt
6292024-12-09T15:18:36.742Zuploading: /tmp/debug/paging.txt (84236 bytes)
6302024-12-09T15:18:36.749Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 84236 -> 85670
6312024-12-09T15:18:37.756Zuploaded: /tmp/debug/paging.txt
6322024-12-09T15:18:37.760Zuploading: /tmp/debug/prstat.txt (892941 bytes)
6332024-12-09T15:18:37.773Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 892941 -> 907184
6342024-12-09T15:18:38.780Zuploaded: /tmp/debug/prstat.txt
6352024-12-09T15:18:38.780Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
6362024-12-09T15:18:39.787Zuploaded: /tmp/debug/psrinfo.txt
6372024-12-09T15:18:39.790Zuploading: /tmp/debug/upstairs-info.txt (92112 bytes)
6382024-12-09T15:18:40.798Zuploaded: /tmp/debug/upstairs-info.txt