01JBJ54771Y9A6TSH88GFT44G4: test-up-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01JBJ54QD4AT68MENE2Z25ZH6Q

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-10-31T20:59:15.387Zjob dependencies complete; ready to run (waiting for 14 m 59 s)
22024-10-31T21:00:21.515Zjob assigned to worker 01JBJ60GP7EEBTZDNRCEPD71PZ (queued for 1 m 6 s)
32024-10-31T21:00:28.527Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-10-31T21:00:29.649Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-10-31T21:00:29.649Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-10-31T21:00:30.752Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-10-31T21:00:30.752Zdownloading input: /input/build/work/bins/crutest.gz
82024-10-31T21:00:31.870Zdownloaded input: /input/build/work/bins/crutest.gz
92024-10-31T21:00:31.870Zdownloading input: /input/build/work/bins/dsc.gz
102024-10-31T21:00:32.543Zdownloaded input: /input/build/work/bins/dsc.gz
112024-10-31T21:00:32.547Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-10-31T21:00:32.558Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-10-31T21:00:32.558Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-10-31T21:00:32.573Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-10-31T21:00:32.573Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-10-31T21:00:32.589Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-10-31T21:00:32.589Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-10-31T21:00:32.611Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-10-31T21:00:32.611Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-10-31T21:00:32.630Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-10-31T21:00:32.630Zdownloading input: /input/build/work/scripts/test_up.sh
222024-10-31T21:00:32.646Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-10-31T21:00:32.647Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-10-31T21:00:32.667Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-10-31T21:00:32.672Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-10-31T21:00:33.431Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-10-31T21:00:33.431Zstarting task 0: "setup"
282024-10-31T21:00:33.440Z++ uname -s
292024-10-31T21:00:33.444Z+ kern=SunOS
302024-10-31T21:00:33.444Z+ case "$kern" in
312024-10-31T21:00:33.444Z+ [[ 0 == 12345 ]]
322024-10-31T21:00:33.445Z+ groupadd -g 12345 build
332024-10-31T21:00:33.448Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
342024-10-31T21:00:35.454Z+ zfs create -o mountpoint=/work rpool/work
352024-10-31T21:00:36.232Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
362024-10-31T21:00:36.235Z+ home_fs=zfs
372024-10-31T21:00:36.235Z+ [[ zfs == autofs ]]
382024-10-31T21:00:36.235Z+ mkdir -p /home/build
392024-10-31T21:00:36.239Z+ chown build:build /home/build /work
402024-10-31T21:00:37.239Z+ chmod 0700 /home/build /work
412024-10-31T21:00:37.242Zprocess exited: duration 3811 ms, exit code 0
 
422024-10-31T21:00:37.248Zstarting task 1: "authentication"
432024-10-31T21:00:37.265Zprocess exited: duration 17 ms, exit code 0
 
442024-10-31T21:00:37.274Zstarting task 2: "build"
452024-10-31T21:00:37.277Z+ banner cores
462024-10-31T21:00:37.296Z
472024-10-31T21:00:37.296Z #### #### ##### ###### ####
482024-10-31T21:00:37.296Z # # # # # # # #
492024-10-31T21:00:37.296Z # # # # # ##### ####
502024-10-31T21:00:37.296Z # # # ##### # #
512024-10-31T21:00:37.296Z # # # # # # # # #
522024-10-31T21:00:37.296Z #### #### # # ###### ####
532024-10-31T21:00:37.296Z
542024-10-31T21:00:37.299Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
552024-10-31T21:00:37.306Z+ echo 'input bins dir contains:'
562024-10-31T21:00:37.306Zinput bins dir contains:
572024-10-31T21:00:37.309Z+ ls -ltr /input/build/work/bins
582024-10-31T21:00:37.309Ztotal 771752
592024-10-31T21:00:37.309Z-rw-r--r-- 1 root root 110753512 Oct 31 21:00 crucible-downstairs.gz
602024-10-31T21:00:37.309Z-rw-r--r-- 1 root root 105892009 Oct 31 21:00 crucible-hammer.gz
612024-10-31T21:00:37.309Z-rw-r--r-- 1 root root 111418362 Oct 31 21:00 crutest.gz
622024-10-31T21:00:37.309Z-rw-r--r-- 1 root root 66803931 Oct 31 21:00 dsc.gz
632024-10-31T21:00:37.311Z+ banner unpack
642024-10-31T21:00:37.311Z
652024-10-31T21:00:37.311Z # # # # ##### ## #### # #
662024-10-31T21:00:37.311Z # # ## # # # # # # # # #
672024-10-31T21:00:37.311Z # # # # # # # # # # ####
682024-10-31T21:00:37.311Z # # # # # ##### ###### # # #
692024-10-31T21:00:37.311Z # # # ## # # # # # # #
702024-10-31T21:00:37.311Z #### # # # # # #### # #
712024-10-31T21:00:37.311Z
722024-10-31T21:00:37.311Z+ mkdir -p /var/tmp/bins
732024-10-31T21:00:37.314Z+ for t in "$input/bins/"*.gz
742024-10-31T21:00:37.314Z++ basename /input/build/work/bins/crucible-downstairs.gz
752024-10-31T21:00:37.316Z+ b=crucible-downstairs.gz
762024-10-31T21:00:37.316Z+ b=crucible-downstairs
772024-10-31T21:00:37.316Z+ gunzip
782024-10-31T21:00:39.610Z+ chmod +x /var/tmp/bins/crucible-downstairs
792024-10-31T21:00:39.612Z+ for t in "$input/bins/"*.gz
802024-10-31T21:00:39.615Z++ basename /input/build/work/bins/crucible-hammer.gz
812024-10-31T21:00:39.616Z+ b=crucible-hammer.gz
822024-10-31T21:00:39.616Z+ b=crucible-hammer
832024-10-31T21:00:39.616Z+ gunzip
842024-10-31T21:00:41.860Z+ chmod +x /var/tmp/bins/crucible-hammer
852024-10-31T21:00:41.862Z+ for t in "$input/bins/"*.gz
862024-10-31T21:00:41.865Z++ basename /input/build/work/bins/crutest.gz
872024-10-31T21:00:41.865Z+ b=crutest.gz
882024-10-31T21:00:41.865Z+ b=crutest
892024-10-31T21:00:41.865Z+ gunzip
902024-10-31T21:00:44.161Z+ chmod +x /var/tmp/bins/crutest
912024-10-31T21:00:44.170Z+ for t in "$input/bins/"*.gz
922024-10-31T21:00:44.170Z++ basename /input/build/work/bins/dsc.gz
932024-10-31T21:00:44.170Z+ b=dsc.gz
942024-10-31T21:00:44.170Z+ b=dsc
952024-10-31T21:00:44.170Z+ gunzip
962024-10-31T21:00:45.580Z+ chmod +x /var/tmp/bins/dsc
972024-10-31T21:00:45.584Z+ export BINDIR=/var/tmp/bins
982024-10-31T21:00:45.584Z+ BINDIR=/var/tmp/bins
992024-10-31T21:00:45.584Z+ jobpid=1137
1002024-10-31T21:00:45.584ZSetup debug logging
1012024-10-31T21:00:45.584Z+ echo 'Setup debug logging'
1022024-10-31T21:00:45.584Z+ mkdir /tmp/debug
1032024-10-31T21:00:45.587Z+ sleep 3600
1042024-10-31T21:00:45.587Z+ psrinfo -v
1052024-10-31T21:00:45.589Z+ df -h
1062024-10-31T21:00:45.612Z+ prstat -d d -mLc 1
1072024-10-31T21:00:45.615Z+ iostat -T d -xn 1
1082024-10-31T21:00:45.615Z+ mpstat -T d 1
1092024-10-31T21:00:45.616Z+ vmstat -T d -p 1
1102024-10-31T21:00:45.618Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1112024-10-31T21:00:45.618Z+ banner test_up_encrypted
1122024-10-31T21:00:45.619Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1132024-10-31T21:00:45.619Z
1142024-10-31T21:00:45.619Z ##### ###### #### ##### # # ##### ###### # #
1152024-10-31T21:00:45.619Z # # # # # # # # # ## #
1162024-10-31T21:00:45.619Z # ##### #### # # # # # ##### # # #
1172024-10-31T21:00:45.619Z # # # # # # ##### # # # #
1182024-10-31T21:00:45.619Z # # # # # # # # # # ##
1192024-10-31T21:00:45.619Z # ###### #### # ####### #### # ####### ###### # #
1202024-10-31T21:00:45.619Z
1212024-10-31T21:00:45.621Z+ ptime -m bash /input/build/work/scripts/test_up.sh -N encrypted
1222024-10-31T21:00:45.631Z/input/build/work
1232024-10-31T21:00:45.659ZTurn off color for downstairs dump
1242024-10-31T21:00:45.705ZUpstairs using key: 4DKnNTLC2CiGL9q/CelH1UA5ZA8Y4npeNsXNUbj4wMc=
1252024-10-31T21:00:45.708Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1262024-10-31T21:00:45.708ZCreating 3 downstairs regions
1272024-10-31T21:00:46.039ZStarting 3 downstairs
1282024-10-31T21:00:46.041Zdsc started at PID: 1183
1292024-10-31T21:00:51.221ZDisable automatic restart on all downstairs
1302024-10-31T21:00:51.275Z
1312024-10-31T21:00:51.276ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1322024-10-31T21:00:51.279ZRunning test: span
1332024-10-31T21:00:55.432ZCompleted test: span
1342024-10-31T21:01:00.440ZRunning test: big
1352024-10-31T21:01:04.846ZCompleted test: big
1362024-10-31T21:01:09.852ZRunning test: dep
1372024-10-31T21:02:03.994ZCompleted test: dep
1382024-10-31T21:02:09.001ZRunning test: balloon
1392024-10-31T21:02:55.373ZCompleted test: balloon
1402024-10-31T21:03:00.381ZRunning test: deactivate
1412024-10-31T21:03:55.164ZCompleted test: deactivate
1422024-10-31T21:04:00.172ZRunning hammer
1432024-10-31T21:04:18.957ZRun repair tests
1442024-10-31T21:04:18.961Z/var/tmp/bins/crutest fill -g 31 -q --key 4DKnNTLC2CiGL9q/CelH1UA5ZA8Y4npeNsXNUbj4wMc= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1452024-10-31T21:04:19.022Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.850641093Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"task":"crutest"}
1462024-10-31T21:04:19.024Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.85124585Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"task":"crutest"}
1472024-10-31T21:04:19.024Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.852951645Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"task":"crutest"}
1482024-10-31T21:04:19.024Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.853173025Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220}
1492024-10-31T21:04:19.024Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"664b05998b747a86e13a12bb6c5e4428458d8bd9\",\n git_commit_timestamp: \"2024-10-31T20:43:19.000000000Z\",\n git_branch: \"alan/crutest-volume-info\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.853223768Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220}
1502024-10-31T21:04:19.024Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.853272672Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220}
1512024-10-31T21:04:19.024Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.853289189Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220}
1522024-10-31T21:04:19.027Z{"msg":"Crucible 4d8896ba-720f-4a6b-a137-ff6fa25813dd has session id: 1733b365-e393-4e0a-b54f-e9bf8cfcc51c","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.853779872Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1532024-10-31T21:04:19.027Z{"msg":"Upstairs opts: Upstairs UUID: 4d8896ba-720f-4a6b-a137-ff6fa25813dd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.853836364Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1542024-10-31T21:04:19.027Z{"msg":"Crucible stats registered with UUID: 4d8896ba-720f-4a6b-a137-ff6fa25813dd","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.853871499Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1552024-10-31T21:04:19.059Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.888354165Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"task":"crutest"}
1562024-10-31T21:04:19.059Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.888409658Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220}
1572024-10-31T21:04:19.059Z{"msg":"4d8896ba-720f-4a6b-a137-ff6fa25813dd active request set","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.888436464Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1582024-10-31T21:04:19.063Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.888453992Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"client":"0","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1592024-10-31T21:04:19.064Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.888518003Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"client":"0","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1602024-10-31T21:04:19.064Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.888550398Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"client":"1","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1612024-10-31T21:04:19.064Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.88861304Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"client":"1","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1622024-10-31T21:04:19.064Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.888651515Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"":"io task","client":"0","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1632024-10-31T21:04:19.064Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.8886852Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"client":"2","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1642024-10-31T21:04:19.064Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.888727064Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"client":"2","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1652024-10-31T21:04:19.064Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.888770238Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"":"io task","client":"2","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1662024-10-31T21:04:19.064Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.888802694Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"":"io task","client":"1","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1672024-10-31T21:04:19.064Z{"msg":"ds_connection connected from Ok(127.0.0.1:51936)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.888833319Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"":"io task","client":"1","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1682024-10-31T21:04:19.064Z{"msg":"ds_connection connected from Ok(127.0.0.1:54582)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.888864365Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"":"io task","client":"0","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1692024-10-31T21:04:19.064Z{"msg":"ds_connection connected from Ok(127.0.0.1:34334)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.889163564Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"":"io task","client":"2","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1702024-10-31T21:04:19.064Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.88984883Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"client":"0","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1712024-10-31T21:04:19.064Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.889887555Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"client":"0","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1722024-10-31T21:04:19.064Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.889906142Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"client":"1","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1732024-10-31T21:04:19.064Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.889932119Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"client":"1","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1742024-10-31T21:04:19.064Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.889969154Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"client":"2","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1752024-10-31T21:04:19.064Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.889999379Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"client":"2","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1762024-10-31T21:04:19.064Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.890558533Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"client":"1","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1772024-10-31T21:04:19.064Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.890685195Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"client":"2","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1782024-10-31T21:04:19.064Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.89072467Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"client":"0","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1792024-10-31T21:04:19.065Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.890924432Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"client":"1","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1802024-10-31T21:04:19.065Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.890973806Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"client":"1","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1812024-10-31T21:04:19.065Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.89101766Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1822024-10-31T21:04:19.065Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.891044406Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"client":"2","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1832024-10-31T21:04:19.065Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.891068723Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"client":"2","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1842024-10-31T21:04:19.065Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.891083241Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1852024-10-31T21:04:19.065Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.891117646Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"client":"0","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1862024-10-31T21:04:19.065Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.891140023Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"client":"0","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1872024-10-31T21:04:19.065Z{"msg":"[0]R flush_numbers: [1681, 1681, 1681, 1681, 1681]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.891156871Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"":"downstairs","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1882024-10-31T21:04:19.065Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.891178308Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"":"downstairs","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1892024-10-31T21:04:19.065Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.891192506Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"":"downstairs","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1902024-10-31T21:04:19.065Z{"msg":"[1]R flush_numbers: [1681, 1681, 1681, 1681, 1681]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.891213283Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"":"downstairs","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1912024-10-31T21:04:19.065Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.891227271Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"":"downstairs","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1922024-10-31T21:04:19.065Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.891249168Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"":"downstairs","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1932024-10-31T21:04:19.065Z{"msg":"[2]R flush_numbers: [1681, 1681, 1681, 1681, 1681]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.891263406Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"":"downstairs","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1942024-10-31T21:04:19.065Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.891277404Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"":"downstairs","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1952024-10-31T21:04:19.065Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.891291002Z","Wait for a query_work_queue command to finish before sending IO
1962024-10-31T21:04:19.065Zhostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"":"downstairs","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1972024-10-31T21:04:19.065Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.891321848Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"":"downstairs","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1982024-10-31T21:04:19.065Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.891335196Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"":"downstairs","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
1992024-10-31T21:04:19.065Z{"msg":"Next flush: 1682","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.891348624Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"":"downstairs","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
2002024-10-31T21:04:19.065Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.891370191Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"":"downstairs","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
2012024-10-31T21:04:19.065Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.891384559Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
2022024-10-31T21:04:19.065Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.891397278Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"client":"0","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
2032024-10-31T21:04:19.065Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.891419104Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"client":"1","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
2042024-10-31T21:04:19.065Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.891432943Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"client":"2","":"downstairs","session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
2052024-10-31T21:04:19.065Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.891445951Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
2062024-10-31T21:04:19.065Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.891458479Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
2072024-10-31T21:04:19.066Z{"msg":"4d8896ba-720f-4a6b-a137-ff6fa25813dd is now active with session: 1733b365-e393-4e0a-b54f-e9bf8cfcc51c","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.891471557Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
2082024-10-31T21:04:19.066Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.891491475Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220,"session_id":"1733b365-e393-4e0a-b54f-e9bf8cfcc51c"}
2092024-10-31T21:04:19.066Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:17.891504493Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220}
2102024-10-31T21:04:19.066ZRegion: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2112024-10-31T21:04:19.066ZFill test
2122024-10-31T21:04:19.109ZRead and Verify all blocks (0..50 range:false)
2132024-10-31T21:04:19.150ZWrote out file "/var/tmp/test_up-build/verify_file"
2142024-10-31T21:04:19.150ZCLIENT: Tests done. All submitted work has been ACK'd
2152024-10-31T21:04:19.152Z----------------------------------------------------------------
2162024-10-31T21:04:19.152Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2172024-10-31T21:04:19.152ZStates: Active Active Active
2182024-10-31T21:04:19.152ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2192024-10-31T21:04:19.152Z 1002 Acked Read 50 Done Done Done false
2202024-10-31T21:04:19.152Z STATES DS:0 DS:1 DS:2 TOTAL
2212024-10-31T21:04:19.152Z Sent 0 0 0 0
2222024-10-31T21:04:19.152Z Done 1 1 1 3
2232024-10-31T21:04:19.152Z Skipped 0 0 0 0
2242024-10-31T21:04:19.153Z Error 0 0 0 0
2252024-10-31T21:04:19.153ZLast Flush: 1001 1001 1001
2262024-10-31T21:04:19.153ZDownstairs last five completed: 1001 1000
2272024-10-31T21:04:19.153ZUpstairs last five completed: 1002 1001 1000
2282024-10-31T21:04:19.153ZCLIENT: Up:0 ds:1 act:3
2292024-10-31T21:04:23.152Z----------------------------------------------------------------
2302024-10-31T21:04:23.152Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2312024-10-31T21:04:23.152ZDownstairs last five completed: 1003 1002 1001 1000
2322024-10-31T21:04:23.152ZUpstairs last five completed: 1003 1002 1001 1000
2332024-10-31T21:04:23.152ZCLIENT: Up:0 ds:0 act:3
2342024-10-31T21:04:23.152ZCLIENT: All crucible jobs finished, exiting program
2352024-10-31T21:04:23.155Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:04:21.980382143Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1220}
2362024-10-31T21:04:23.163ZRepair setup passed
2372024-10-31T21:04:23.167ZCopy the region for /var/tmp/test_up-build/8830
2382024-10-31T21:04:23.169Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2392024-10-31T21:04:23.172Z/var/tmp/bins/crutest fill -g 32 -q --key 4DKnNTLC2CiGL9q/CelH1UA5ZA8Y4npeNsXNUbj4wMc= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2402024-10-31T21:04:23.234Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.060417635Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"task":"crutest"}
2412024-10-31T21:04:23.237Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.061044709Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"task":"crutest"}
2422024-10-31T21:04:23.237Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.062731437Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"task":"crutest"}
2432024-10-31T21:04:23.237Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.062944238Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226}
2442024-10-31T21:04:23.237Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"664b05998b747a86e13a12bb6c5e4428458d8bd9\",\n git_commit_timestamp: \"2024-10-31T20:43:19.000000000Z\",\n git_branch: \"alan/crutest-volume-info\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.062985472Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226}
2452024-10-31T21:04:23.237Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.063009649Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226}
2462024-10-31T21:04:23.237Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.063026167Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226}
2472024-10-31T21:04:23.237Z{"msg":"Crucible ca3c584b-104b-4b34-8703-300b6af4b212 has session id: 9acde2c2-b8d6-40e9-a4cf-7dc68a088dab","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.063532297Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2482024-10-31T21:04:23.237Z{"msg":"Upstairs opts: Upstairs UUID: ca3c584b-104b-4b34-8703-300b6af4b212, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.063577601Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2492024-10-31T21:04:23.237Z{"msg":"Crucible stats registered with UUID: ca3c584b-104b-4b34-8703-300b6af4b212","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.063594729Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2502024-10-31T21:04:23.270Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.098390572Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"task":"crutest"}
2512024-10-31T21:04:23.270Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.098454773Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226}
2522024-10-31T21:04:23.275Z{"msg":"ca3c584b-104b-4b34-8703-300b6af4b212 active request set","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.098485849Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2532024-10-31T21:04:23.275Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.098528843Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"client":"0","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2542024-10-31T21:04:23.275Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.098596364Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"client":"0","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2552024-10-31T21:04:23.275Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.098635888Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"client":"1","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2562024-10-31T21:04:23.275Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.098687921Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"client":"1","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2572024-10-31T21:04:23.275Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.098748413Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"client":"2","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2582024-10-31T21:04:23.275Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.098788527Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"client":"2","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2592024-10-31T21:04:23.275Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.098815944Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"":"io task","client":"2","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2602024-10-31T21:04:23.275Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.098848609Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"":"io task","client":"0","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2612024-10-31T21:04:23.275Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.098872176Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"":"io task","client":"1","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2622024-10-31T21:04:23.276Z{"msg":"ds_connection connected from Ok(127.0.0.1:65300)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.098903741Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"":"io task","client":"0","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2632024-10-31T21:04:23.276Z{"msg":"ds_connection connected from Ok(127.0.0.1:61207)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.098928838Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"":"io task","client":"2","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2642024-10-31T21:04:23.276Z{"msg":"ds_connection connected from Ok(127.0.0.1:57643)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.099089156Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"":"io task","client":"1","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2652024-10-31T21:04:23.276Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.099691303Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"client":"1","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2662024-10-31T21:04:23.276Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.099729578Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"client":"1","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2672024-10-31T21:04:23.276Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.099748145Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"client":"2","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2682024-10-31T21:04:23.276Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.099762203Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"client":"2","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2692024-10-31T21:04:23.276Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.099796079Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"client":"0","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2702024-10-31T21:04:23.276Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.099816236Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"client":"0","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2712024-10-31T21:04:23.276Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.100426172Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"client":"2","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2722024-10-31T21:04:23.276Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.100477165Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"client":"1","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2732024-10-31T21:04:23.276Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.100581811Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"client":"0","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2742024-10-31T21:04:23.276Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.100912716Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"client":"1","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2752024-10-31T21:04:23.276Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.100962579Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"client":"1","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2762024-10-31T21:04:23.276Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.101042668Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2772024-10-31T21:04:23.276Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.101077923Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"client":"2","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2782024-10-31T21:04:23.276Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.101094691Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"client":"2","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2792024-10-31T21:04:23.276Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.101116468Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2802024-10-31T21:04:23.279Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.101132386Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"client":"0","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2812024-10-31T21:04:23.279Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.10116793Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"client":"0","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2822024-10-31T21:04:23.279Z{"msg":"[0]R flush_numbers: [1682, 1682, 1682, 1682, 1682]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.101197397Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"":"downstairs","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2832024-10-31T21:04:23.280Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.101215874Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"":"downstairs","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2842024-10-31T21:04:23.280Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.101234931Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
2852024-10-31T21:04:23.280Z9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2862024-10-31T21:04:23.280Z{"msg":"[1]R flush_numbers: [1682, 1682, 1682, 1682, 1682]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.101294623Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"":"downstairs","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2872024-10-31T21:04:23.280Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.101327689Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"":"downstairs","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2882024-10-31T21:04:23.280Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.101348896Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"":"downstairs","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2892024-10-31T21:04:23.280Z{"msg":"[2]R flush_numbers: [1682, 1682, 1682, 1682, 1682]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.101385881Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"":"downstairs","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2902024-10-31T21:04:23.280Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.101418716Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"":"downstairs","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2912024-10-31T21:04:23.280Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.101436804Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"":"downstairs","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2922024-10-31T21:04:23.280Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.101471659Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"":"downstairs","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2932024-10-31T21:04:23.280Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.101506014Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"":"downstairs","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2942024-10-31T21:04:23.280Z{"msg":"Next flush: 1683","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.101522252Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"":"downstairs","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2952024-10-31T21:04:23.280Z{"msg":"All extents match","v":0,"name":"crucible","level":30Region: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2962024-10-31T21:04:23.280Z,"time":"2024-10-31T21:04:22.101555197Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"":"downstairs","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2972024-10-31T21:04:23.280Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.101584224Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2982024-10-31T21:04:23.280Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.101615399Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"client":"0","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
2992024-10-31T21:04:23.280Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.101650084Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"client":"1","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
3002024-10-31T21:04:23.280Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.101686769Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"client":"2","":"downstairs","session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
3012024-10-31T21:04:23.280Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.101723494Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
3022024-10-31T21:04:23.280Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.1017581Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
3032024-10-31T21:04:23.280Z{"msg":"ca3c584b-104b-4b34-8703-300b6af4b212 is now active with session: 9acde2c2-b8d6-40e9-a4cf-7dc68a088dab","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.101793245Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
3042024-10-31T21:04:23.280ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
3052024-10-31T21:04:23.280Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.101833919Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226,"session_id":"9acde2c2-b8d6-40e9-a4cf-7dc68a088dab"}
3062024-10-31T21:04:23.281ZFill test
3072024-10-31T21:04:23.281Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:22.101869834Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226}
3082024-10-31T21:04:23.321ZRead and Verify all blocks (0..50 range:false)
3092024-10-31T21:04:23.361ZWrote out file "/var/tmp/test_up-build/verify_file"
3102024-10-31T21:04:23.361ZCLIENT: Tests done. All submitted work has been ACK'd
3112024-10-31T21:04:23.364Z----------------------------------------------------------------
3122024-10-31T21:04:23.364Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
3132024-10-31T21:04:23.364ZStates: Active Active Active
3142024-10-31T21:04:23.364ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
3152024-10-31T21:04:23.364Z 1002 Acked Read 50 Done Done Done false
3162024-10-31T21:04:23.364Z STATES DS:0 DS:1 DS:2 TOTAL
3172024-10-31T21:04:23.364Z Sent 0 0 0 0
3182024-10-31T21:04:23.364Z Done 1 1 1 3
3192024-10-31T21:04:23.364Z Skipped 0 0 0 0
3202024-10-31T21:04:23.364Z Error 0 0 0 0
3212024-10-31T21:04:23.364ZLast Flush: 1001 1001 1001
3222024-10-31T21:04:23.364ZDownstairs last five completed: 1001 1000
3232024-10-31T21:04:23.364ZUpstairs last five completed: 1002 1001 1000
3242024-10-31T21:04:23.364ZCLIENT: Up:0 ds:1 act:3
3252024-10-31T21:04:27.364Z----------------------------------------------------------------
3262024-10-31T21:04:27.364Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
3272024-10-31T21:04:27.364ZDownstairs last five completed: 1003 1002 1001 1000
3282024-10-31T21:04:27.364ZUpstairs last five completed: 1003 1002 1001 1000
3292024-10-31T21:04:27.364ZCLIENT: Up:0 ds:0 act:3
3302024-10-31T21:04:27.364ZCLIENT: All crucible jobs finished, exiting program
3312024-10-31T21:04:27.367Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:04:26.191980674Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1226}
3322024-10-31T21:04:27.371ZRepair part 1 passed
3332024-10-31T21:04:27.374Z
3342024-10-31T21:04:27.374ZKill the current downstairs
3352024-10-31T21:04:27.542ZDownstairs 2 stopped
3362024-10-31T21:04:27.542Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
3372024-10-31T21:04:27.545ZNow put back the original so we have a mismatch
3382024-10-31T21:04:27.545Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
3392024-10-31T21:04:27.547ZRestart downstairs with old directory
3402024-10-31T21:04:27.607Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
3412024-10-31T21:04:27.623ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3422024-10-31T21:04:27.625Z 0 000-009 32 32 31 1683 1683 1682 F F F <---
3432024-10-31T21:04:27.625Z 1 010-019 32 32 31 1683 1683 1682 F F F <---
3442024-10-31T21:04:27.625Z 2 020-029 32 32 31 1683 1683 1682 F F F <---
3452024-10-31T21:04:27.625Z 3 030-039 32 32 31 1683 1683 1682 F F F <---
3462024-10-31T21:04:27.625Z 4 040-049 32 32 31 1683 1683 1682 F F F <---
3472024-10-31T21:04:27.625ZMax gen: 32, Max flush: 1683
3482024-10-31T21:04:27.625ZError: Difference in extent metadata found!
3492024-10-31T21:04:27.627Zdump test found error as expected
3502024-10-31T21:04:27.627Z
3512024-10-31T21:04:27.627Z
3522024-10-31T21:04:27.628Z/var/tmp/bins/crutest verify --range -g 33 -q --key 4DKnNTLC2CiGL9q/CelH1UA5ZA8Y4npeNsXNUbj4wMc= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
3532024-10-31T21:04:27.688Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.515297497Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"task":"crutest"}
3542024-10-31T21:04:27.691Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.515852781Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"task":"crutest"}
3552024-10-31T21:04:27.691Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.517598431Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"task":"crutest"}
3562024-10-31T21:04:27.691Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.517836109Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234}
3572024-10-31T21:04:27.691Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"664b05998b747a86e13a12bb6c5e4428458d8bd9\",\n git_commit_timestamp: \"2024-10-31T20:43:19.000000000Z\",\n git_branch: \"alan/crutest-volume-info\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.517876563Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234}
3582024-10-31T21:04:27.691Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.517892451Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234}
3592024-10-31T21:04:27.691Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.517908159Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234}
3602024-10-31T21:04:27.691Z{"msg":"Crucible 1a6e62dd-64cb-498d-bd19-2c77fe96c5cb has session id: 4b3bb0b1-2951-4108-88e2-b6beaf20cb1a","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.518414159Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3612024-10-31T21:04:27.691Z{"msg":"Upstairs opts: Upstairs UUID: 1a6e62dd-64cb-498d-bd19-2c77fe96c5cb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.518468602Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3622024-10-31T21:04:27.691Z{"msg":"Crucible stats registered with UUID: 1a6e62dd-64cb-498d-bd19-2c77fe96c5cb","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.518496738Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3632024-10-31T21:04:27.725Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.553196834Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"task":"crutest"}
3642024-10-31T21:04:27.726Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.553262605Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234}
3652024-10-31T21:04:27.729Z{"msg":"1a6e62dd-64cb-498d-bd19-2c77fe96c5cb active request set","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.553293611Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3662024-10-31T21:04:27.729Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.553339634Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"client":"0","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3672024-10-31T21:04:27.729Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.553415074Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"client":"0","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3682024-10-31T21:04:27.729Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.553459568Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"client":"1","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3692024-10-31T21:04:27.729Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.553489264Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"client":"1","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3702024-10-31T21:04:27.729Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.55351709Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"client":"2","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3712024-10-31T21:04:27.729Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.553546586Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"client":"2","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3722024-10-31T21:04:27.729Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.55358884Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"":"io task","client":"2","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3732024-10-31T21:04:27.729Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.553620876Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"":"io task","client":"0","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3742024-10-31T21:04:27.729Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.553655491Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"":"io task","client":"1","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3752024-10-31T21:04:27.729Z{"msg":"ds_connection connected from Ok(127.0.0.1:65403)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.553690806Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"":"io task","client":"0","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3762024-10-31T21:04:27.729Z{"msg":"ds_connection connected from Ok(127.0.0.1:58151)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.553725772Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"":"io task","client":"1","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3772024-10-31T21:04:27.729Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:04:26.553764316Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"":"io task","client":"2","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3782024-10-31T21:04:27.729Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:04:26.553846895Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"":"io task","client":"2","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3792024-10-31T21:04:27.729Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:04:26.553911006Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3802024-10-31T21:04:27.729Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.553952601Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"client":"2","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3812024-10-31T21:04:27.729Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.553993565Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"client":"2","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3822024-10-31T21:04:27.729Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.554023651Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"":"io task","client":"2","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3832024-10-31T21:04:27.729Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.554389841Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"client":"0","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3842024-10-31T21:04:27.729Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.554427785Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"client":"0","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3852024-10-31T21:04:27.729Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.554555468Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"client":"1","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3862024-10-31T21:04:27.729Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.554588813Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"client":"1","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3872024-10-31T21:04:27.729Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.554941665Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"client":"0","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3882024-10-31T21:04:27.729Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.555070557Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"client":"1","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3892024-10-31T21:04:27.729Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.555288757Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"client":"0","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3902024-10-31T21:04:27.729Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.555318713Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"client":"0","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3912024-10-31T21:04:27.729Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.555334301Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3922024-10-31T21:04:27.729Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.555448675Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"client":"1","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3932024-10-31T21:04:27.730Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.555477071Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"client":"1","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3942024-10-31T21:04:27.730Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:26.555491869Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3952024-10-31T21:04:37.729Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.55549921Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"":"io task","client":"2","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3962024-10-31T21:04:37.729Z{"msg":"ds_connection connected from Ok(127.0.0.1:38064)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.555602986Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"":"io task","client":"2","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3972024-10-31T21:04:37.734Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.556563214Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"client":"2","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3982024-10-31T21:04:37.734Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.556605578Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"client":"2","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
3992024-10-31T21:04:37.734Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.557186408Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"client":"2","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4002024-10-31T21:04:37.734Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.557591953Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"client":"2","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4012024-10-31T21:04:37.734Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.557623398Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"client":"2","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4022024-10-31T21:04:37.734Z{"msg":"[0]R flush_numbers: [1683, 1683, 1683, 1683, 1683]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.557639756Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4032024-10-31T21:04:37.735Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.557663353Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4042024-10-31T21:04:37.735Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.557678351Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4052024-10-31T21:04:37.735Z{"msg":"[1]R flush_numbers: [1683, 1683, 1683, 1683, 1683]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.557692699Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4062024-10-31T21:04:37.735Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.557706117Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4072024-10-31T21:04:37.735Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.557720825Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4082024-10-31T21:04:37.735Z{"msg":"[2]R flush_numbers: [1682, 1682, 1682, 1682, 1682]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.557743962Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4092024-10-31T21:04:37.735Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.55775933Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4102024-10-31T21:04:37.735Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.557781627Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4112024-10-31T21:04:37.735Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.557796045Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4122024-10-31T21:04:37.735Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.557817262Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4132024-10-31T21:04:37.735Z{"msg":"Next flush: 1684","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.55783117Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4142024-10-31T21:04:37.735Z{"msg":"Extent 0 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.557852997Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4152024-10-31T21:04:37.735Z{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.557868125Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4162024-10-31T21:04:37.735Z{"msg":"extent:0 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.557890102Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4172024-10-31T21:04:37.735Z{"msg":"extent:0 flush: 1683 1683 1682 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.557905519Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4182024-10-31T21:04:37.735Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.557927556Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4192024-10-31T21:04:37.735Z{"msg":"extent:0 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.557942454Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4202024-10-31T21:04:37.735Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.557956823Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4212024-10-31T21:04:37.735Z{"msg":"find dest for source 0 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.557979689Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4222024-10-31T21:04:37.735Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.557994547Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4232024-10-31T21:04:37.735Z{"msg":"Extent 1 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558016024Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4242024-10-31T21:04:37.735Z{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558039151Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4252024-10-31T21:04:37.735Z{"msg":"extent:1 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558054999Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4262024-10-31T21:04:37.735Z{"msg":"extent:1 flush: 1683 1683 1682 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558077586Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4272024-10-31T21:04:37.735Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558092714Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4282024-10-31T21:04:37.735Z{"msg":"extent:1 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558115231Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4292024-10-31T21:04:37.735Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558130279Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4302024-10-31T21:04:37.735Z{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558152416Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4312024-10-31T21:04:37.735Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558167963Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4322024-10-31T21:04:37.736Z{"msg":"Extent 2 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558190471Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4332024-10-31T21:04:37.736Z{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558204818Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4342024-10-31T21:04:37.736Z{"msg":"extent:2 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558227095Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4352024-10-31T21:04:37.736Z{"msg":"extent:2 flush: 1683 1683 1682 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558242383Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4362024-10-31T21:04:37.736Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.55826485Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4372024-10-31T21:04:37.736Z{"msg":"extent:2 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558279768Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4382024-10-31T21:04:37.736Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558302525Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4392024-10-31T21:04:37.736Z{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558317573Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4402024-10-31T21:04:37.736Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.55833926Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4412024-10-31T21:04:37.736Z{"msg":"Extent 3 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558354318Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4422024-10-31T21:04:37.736Z{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558375675Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4432024-10-31T21:04:37.736Z{"msg":"extent:3 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558390683Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4442024-10-31T21:04:37.736Z{"msg":"extent:3 flush: 1683 1683 1682 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.55841349Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4452024-10-31T21:04:37.736Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558429628Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4462024-10-31T21:04:37.736Z{"msg":"extent:3 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558452244Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4472024-10-31T21:04:37.736Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558467992Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4482024-10-31T21:04:37.736Z{"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558490309Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4492024-10-31T21:04:37.736Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558505117Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4502024-10-31T21:04:37.736Z{"msg":"Extent 4 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558536573Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4512024-10-31T21:04:37.736Z{"msg":"First source client ID for extent 4","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558563979Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4522024-10-31T21:04:37.740Z{"msg":"extent:4 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558601304Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4532024-10-31T21:04:37.740Z{"msg":"extent:4 flush: 1683 1683 1682 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558638999Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4542024-10-31T21:04:37.740Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558661966Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4552024-10-31T21:04:37.740Z{"msg":"extent:4 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558694051Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4562024-10-31T21:04:37.740Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558713399Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4572024-10-31T21:04:37.740Z{"msg":"find dest for source 0 for extent at index 4","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558748724Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4582024-10-31T21:04:37.740Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558785059Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4592024-10-31T21:04:37.740Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558820904Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"client":"0","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4602024-10-31T21:04:37.740Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558841511Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"client":"1","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4612024-10-31T21:04:37.740Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558872367Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"client":"2","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4622024-10-31T21:04:37.741Z{"msg":"starting reconciliation a5e01d36-74e4-47d3-b40c-1153b7d2b8ac: found 5 extents that need repair","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558889684Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4632024-10-31T21:04:37.741Z{"msg":"Full repair list: {ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.558932558Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4642024-10-31T21:04:37.741Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 1684, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 1684, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 1684, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 1684, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 1684, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.559075339Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4652024-10-31T21:04:37.741Z{"msg":"no Nexus client from DNS, aborting start notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:04:36.559127662Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"reconcile":"a5e01d36-74e4-47d3-b40c-1153b7d2b8ac","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4662024-10-31T21:04:37.741Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:04:36.559149429Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"reconcile":"a5e01d36-74e4-47d3-b40c-1153b7d2b8ac","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4672024-10-31T21:04:37.741Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:04:36.559178905Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"reconcile":"a5e01d36-74e4-47d3-b40c-1153b7d2b8ac","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4682024-10-31T21:04:37.741Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.559349751Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"client":"2","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4692024-10-31T21:04:37.741Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:04:36.559381857Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"reconcile":"a5e01d36-74e4-47d3-b40c-1153b7d2b8ac","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4702024-10-31T21:04:37.745Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:04:36.571260846Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"reconcile":"a5e01d36-74e4-47d3-b40c-1153b7d2b8ac","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4712024-10-31T21:04:37.748Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:04:36.571683008Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"reconcile":"a5e01d36-74e4-47d3-b40c-1153b7d2b8ac","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4722024-10-31T21:04:37.748Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:04:36.572007014Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"reconcile":"a5e01d36-74e4-47d3-b40c-1153b7d2b8ac","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4732024-10-31T21:04:37.748Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.572280906Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"client":"2","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4742024-10-31T21:04:37.748Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:04:36.572309862Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"reconcile":"a5e01d36-74e4-47d3-b40c-1153b7d2b8ac","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4752024-10-31T21:04:37.753Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:04:36.579346646Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"reconcile":"a5e01d36-74e4-47d3-b40c-1153b7d2b8ac","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4762024-10-31T21:04:37.755Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:04:36.579810812Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"reconcile":"a5e01d36-74e4-47d3-b40c-1153b7d2b8ac","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4772024-10-31T21:04:37.755Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:04:36.580123649Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"reconcile":"a5e01d36-74e4-47d3-b40c-1153b7d2b8ac","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4782024-10-31T21:04:37.755Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.58041158Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"client":"2","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4792024-10-31T21:04:37.756Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:04:36.580452034Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"reconcile":"a5e01d36-74e4-47d3-b40c-1153b7d2b8ac","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4802024-10-31T21:04:37.761Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:04:36.58769726Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"reconcile":"a5e01d36-74e4-47d3-b40c-1153b7d2b8ac","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4812024-10-31T21:04:37.765Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:04:36.588054141Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"reconcile":"a5e01d36-74e4-47d3-b40c-1153b7d2b8ac","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4822024-10-31T21:04:37.765Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:04:36.58835218Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"reconcile":"a5e01d36-74e4-47d3-b40c-1153b7d2b8ac","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4832024-10-31T21:04:37.765Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.588585108Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"client":"2","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4842024-10-31T21:04:37.765Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:04:36.588634981Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"reconcile":"a5e01d36-74e4-47d3-b40c-1153b7d2b8ac","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4852024-10-31T21:04:37.771Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:04:36.596272172Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"reconcile":"a5e01d36-74e4-47d3-b40c-1153b7d2b8ac","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4862024-10-31T21:04:37.774Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:04:36.596703633Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"reconcile":"a5e01d36-74e4-47d3-b40c-1153b7d2b8ac","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4872024-10-31T21:04:37.774Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:04:36.596977366Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"reconcile":"a5e01d36-74e4-47d3-b40c-1153b7d2b8ac","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4882024-10-31T21:04:37.774Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.597284544Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"client":"2","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4892024-10-31T21:04:37.774Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:04:36.597331227Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"reconcile":"a5e01d36-74e4-47d3-b40c-1153b7d2b8ac","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4902024-10-31T21:04:37.778Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:04:36.604651652Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"reconcile":"a5e01d36-74e4-47d3-b40c-1153b7d2b8ac","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4912024-10-31T21:04:37.781Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.604972068Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4922024-10-31T21:04:37.781Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.605013642Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"client":"0","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4932024-10-31T21:04:37.781Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.605034809Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"client":"1","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4942024-10-31T21:04:37.781Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.605071924Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"client":"2","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4952024-10-31T21:04:37.781Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.605089562Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4962024-10-31T21:04:37.781Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.605119588Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4972024-10-31T21:04:37.781Z{"msg":"1a6e62dd-64cb-498d-bd19-2c77fe96c5cb is now active with session: 4b3bb0b1-2951-4108-88e2-b6beaf20cb1a","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:04:36.605136675Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
4982024-10-31T21:04:37.781Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO
4992024-10-31T21:04:37.781Ztime":"2024-10-31T21:04:36.605155013Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234}
5002024-10-31T21:04:37.781Z{"msg":"no Nexus client from DNS, aborting finish notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:04:36.605227053Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234,"reconcile":"a5e01d36-74e4-47d3-b40c-1153b7d2b8ac","":"downstairs","":"downstairs","session_id":"4b3bb0b1-2951-4108-88e2-b6beaf20cb1a"}
5012024-10-31T21:04:37.781ZRegion: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
5022024-10-31T21:04:37.781ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
5032024-10-31T21:04:37.781ZRead and Verify all blocks (0..50 range:true)
5042024-10-31T21:04:37.820ZWrote out file "/var/tmp/test_up-build/verify_file"
5052024-10-31T21:04:37.820ZVerify test completed
5062024-10-31T21:04:37.823ZWrote out file "/var/tmp/test_up-build/verify_file"
5072024-10-31T21:04:37.823ZCLIENT: Tests done. All submitted work has been ACK'd
5082024-10-31T21:04:37.823Z----------------------------------------------------------------
5092024-10-31T21:04:37.823Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
5102024-10-31T21:04:37.823ZStates: Active Active Active
5112024-10-31T21:04:37.823ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
5122024-10-31T21:04:37.823Z 1000 Acked Read 50 Done Done Done false
5132024-10-31T21:04:37.823Z STATES DS:0 DS:1 DS:2 TOTAL
5142024-10-31T21:04:37.823Z Sent 0 0 0 0
5152024-10-31T21:04:37.823Z Done 1 1 1 3
5162024-10-31T21:04:37.823Z Skipped 0 0 0 0
5172024-10-31T21:04:37.823Z Error 0 0 0 0
5182024-10-31T21:04:37.823ZLast Flush: 0 0 0
5192024-10-31T21:04:37.823ZDownstairs last five completed:
5202024-10-31T21:04:37.823ZUpstairs last five completed: 1000
5212024-10-31T21:04:37.823ZCLIENT: Up:0 ds:1 act:3
5222024-10-31T21:04:41.824Z----------------------------------------------------------------
5232024-10-31T21:04:41.824Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
5242024-10-31T21:04:41.824ZDownstairs last five completed: 1001 1000
5252024-10-31T21:04:41.824ZUpstairs last five completed: 1001 1000
5262024-10-31T21:04:41.824ZCLIENT: Up:0 ds:0 act:3
5272024-10-31T21:04:41.824ZCLIENT: All crucible jobs finished, exiting program
5282024-10-31T21:04:41.826Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:04:40.649418517Z","hostname":"w-01JBJ60GP7EEBTZDNRCEPD71PZ","pid":1234}
5292024-10-31T21:04:41.832ZRepair part 2 passed
5302024-10-31T21:04:41.834Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
5312024-10-31T21:04:41.849ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
5322024-10-31T21:04:41.849Z 0 000-009 32 32 32 1683 1683 1683 F F F
5332024-10-31T21:04:41.849Z 1 010-019 32 32 32 1683 1683 1683 F F F
5342024-10-31T21:04:41.849Z 2 020-029 32 32 32 1683 1683 1683 F F F
5352024-10-31T21:04:41.849Z 3 030-039 32 32 32 1683 1683 1683 F F F
5362024-10-31T21:04:41.849Z 4 040-049 32 32 32 1683 1683 1683 F F F
5372024-10-31T21:04:41.849ZMax gen: 32, Max flush: 1683
5382024-10-31T21:04:41.852Zdump test passed
5392024-10-31T21:04:41.855Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
5402024-10-31T21:04:41.868Z Extent 1
5412024-10-31T21:04:41.868ZGEN 32 32 32
5422024-10-31T21:04:41.869ZFLUSH_ID 1683 1683 1683
5432024-10-31T21:04:41.871ZDIRTY
5442024-10-31T21:04:41.871Z
5452024-10-31T21:04:41.871ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
5462024-10-31T21:04:41.871Z010 A A A A A A
5472024-10-31T21:04:41.871Z011 A A A A A A
5482024-10-31T21:04:41.873Z012 A A A A A A
5492024-10-31T21:04:41.874Z013 A A A A A A
5502024-10-31T21:04:41.876Z014 A A A A A A
5512024-10-31T21:04:41.876Z015 A A A A A A
5522024-10-31T21:04:41.879Z016 A A A A A A
5532024-10-31T21:04:41.879Z017 A A A A A A
5542024-10-31T21:04:41.881Z018 A A A A A A
5552024-10-31T21:04:41.881Z019 A A A A A A
5562024-10-31T21:04:41.884Zdump extent test passed
5572024-10-31T21:04:41.886Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
5582024-10-31T21:04:41.900ZExtent 2 Block in extent 0 Actual block 20
5592024-10-31T21:04:41.900Z
5602024-10-31T21:04:41.902Z DATA SHA256 VER
5612024-10-31T21:04:41.902Z------ ---------------------------------------------------------------- ---
5622024-10-31T21:04:41.905Z 0 a7a035b14c25f6fcc3ef24c6e5fbeafb21fa5bf1d389a4e85511a5db8b1c209e A
5632024-10-31T21:04:41.905Z 1 a7a035b14c25f6fcc3ef24c6e5fbeafb21fa5bf1d389a4e85511a5db8b1c209e A
5642024-10-31T21:04:41.905Z 2 a7a035b14c25f6fcc3ef24c6e5fbeafb21fa5bf1d389a4e85511a5db8b1c209e A
5652024-10-31T21:04:41.905Z
5662024-10-31T21:04:41.905ZNONCES 0 1 2 DIFF
5672024-10-31T21:04:41.905Z------ ------------------------ ------------------------ ------------------------ -----
5682024-10-31T21:04:41.905Z 0 a8ffb10d39fd7d55f18427f6 a8ffb10d39fd7d55f18427f6 a8ffb10d39fd7d55f18427f6
5692024-10-31T21:04:41.906Z
5702024-10-31T21:04:41.906Z TAGS 0 1 2 DIFF
5712024-10-31T21:04:41.906Z------ -------------------------------- -------------------------------- -------------------------------- -----
5722024-10-31T21:04:41.906Z 0 cd81f8639c3f8a188ebcd0c9345ddc24 cd81f8639c3f8a188ebcd0c9345ddc24 cd81f8639c3f8a188ebcd0c9345ddc24
5732024-10-31T21:04:41.906Z
5742024-10-31T21:04:41.906ZHASHES 0 1 2 DIFF
5752024-10-31T21:04:41.906Z------ ---------------- ---------------- ---------------- -----
5762024-10-31T21:04:41.906Z
5772024-10-31T21:04:41.908Zdump block test passed
5782024-10-31T21:04:41.908ZInitial upstairs tests have completed, stopping all downstairs
5792024-10-31T21:04:46.973ZCreating 4 larger downstairs regions
5802024-10-31T21:04:47.441ZStarting 4 downstairs
5812024-10-31T21:04:52.449Zdsc restarted at PID: 1247
5822024-10-31T21:04:52.681ZNow do the replace-reconcile test
5832024-10-31T21:04:52.736ZUsing 8840 for the replacement port
5842024-10-31T21:08:04.683ZCompleted test: replace-reconcile
5852024-10-31T21:08:04.686ZNow do the replace-before-active test
5862024-10-31T21:10:51.598ZCompleted test: replace-before-active
5872024-10-31T21:10:51.598ZAll tests have completed, stopping all downstairs
5882024-10-31T21:10:51.658Z
5892024-10-31T21:10:51.659ZAll Tests have passed
5902024-10-31T21:10:51.677Z10:06 Test duration
5912024-10-31T21:10:51.677Z
5922024-10-31T21:10:51.677Zreal 10:05.935529387
5932024-10-31T21:10:51.677Zuser 6:36.219453610
5942024-10-31T21:10:51.677Zsys 14.161495841
5952024-10-31T21:10:51.677Ztrap 0.247996060
5962024-10-31T21:10:51.677Ztflt 0.012811248
5972024-10-31T21:10:51.677Zdflt 0.092914662
5982024-10-31T21:10:51.678Zkflt 0.000922664
5992024-10-31T21:10:51.678Zlock 8:56:37.322732738
6002024-10-31T21:10:51.678Zslp 35:20.792892243
6012024-10-31T21:10:51.678Zlat 25.936931096
6022024-10-31T21:10:51.678Zstop 0.141269170
6032024-10-31T21:10:51.678Z+ echo 'test-up-encrypted ends'
6042024-10-31T21:10:51.678Ztest-up-encrypted ends
6052024-10-31T21:10:56.662Zprocess exited: duration 614286 ms, exit code 0
6062024-10-31T21:10:56.662Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
6072024-10-31T21:11:56.710Zexec warning: : stdout descriptor may be held open by a background process; giving up!
6082024-10-31T21:11:56.710Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
6092024-10-31T21:11:56.722Zfound 14 output files
6102024-10-31T21:11:56.722Zuploading: /tmp/test_up-build/dsc-out.txt (41534 bytes)
6112024-10-31T21:11:57.733Zuploaded: /tmp/test_up-build/dsc-out.txt
6122024-10-31T21:11:57.733Zuploading: /tmp/test_up-build/test_up_out.txt (10189920 bytes)
6132024-10-31T21:11:58.848Zuploaded: /tmp/test_up-build/test_up_out.txt
6142024-10-31T21:11:58.848Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (253729 bytes)
6152024-10-31T21:11:59.861Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
6162024-10-31T21:11:59.864Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (144641 bytes)
6172024-10-31T21:12:00.875Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
6182024-10-31T21:12:00.876Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (251114 bytes)
6192024-10-31T21:12:01.896Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
6202024-10-31T21:12:01.896Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (139442 bytes)
6212024-10-31T21:12:02.910Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
6222024-10-31T21:12:02.914Zuploading: /tmp/debug/df.txt (1270 bytes)
6232024-10-31T21:12:03.921Zuploaded: /tmp/debug/df.txt
6242024-10-31T21:12:03.924Zuploading: /tmp/debug/dtrace.txt (581267 bytes)
6252024-10-31T21:12:04.938Zuploaded: /tmp/debug/dtrace.txt
6262024-10-31T21:12:04.940Zuploading: /tmp/debug/iostat.txt (201382 bytes)
6272024-10-31T21:12:04.946Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 201382 -> 204082
6282024-10-31T21:12:05.952Zuploaded: /tmp/debug/iostat.txt
6292024-10-31T21:12:05.955Zuploading: /tmp/debug/mpstat.txt (489164 bytes)
6302024-10-31T21:12:05.965Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 489164 -> 496454
6312024-10-31T21:12:06.971Zuploaded: /tmp/debug/mpstat.txt
6322024-10-31T21:12:06.971Zuploading: /tmp/debug/paging.txt (87909 bytes)
6332024-10-31T21:12:06.978Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 87909 -> 89332
6342024-10-31T21:12:07.984Zuploaded: /tmp/debug/paging.txt
6352024-10-31T21:12:07.987Zuploading: /tmp/debug/prstat.txt (946921 bytes)
6362024-10-31T21:12:08.003Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 946921 -> 961139
6372024-10-31T21:12:09.009Zuploaded: /tmp/debug/prstat.txt
6382024-10-31T21:12:09.012Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
6392024-10-31T21:12:10.017Zuploaded: /tmp/debug/psrinfo.txt
6402024-10-31T21:12:10.023Zuploading: /tmp/debug/upstairs-info.txt (104280 bytes)
6412024-10-31T21:12:11.032Zuploaded: /tmp/debug/upstairs-info.txt