01J8TTN3WH0E2FXMTQNQRK6SER: test-up-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01J8TTNGYSWZHQ0RFCC2VM14Z4

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-09-27T23:03:10.450Zjob dependencies complete; ready to run (waiting for 15 m 53 s)
22024-09-27T23:04:00.725Zjob assigned to worker 01J8TVJQ3EWQ3ZCSF08GFSP7EF (queued for 50 s)
32024-09-27T23:04:09.446Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-09-27T23:04:10.653Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-09-27T23:04:10.654Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-09-27T23:04:11.840Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-09-27T23:04:11.840Zdownloading input: /input/build/work/bins/crutest.gz
82024-09-27T23:04:13.096Zdownloaded input: /input/build/work/bins/crutest.gz
92024-09-27T23:04:13.096Zdownloading input: /input/build/work/bins/dsc.gz
102024-09-27T23:04:13.777Zdownloaded input: /input/build/work/bins/dsc.gz
112024-09-27T23:04:13.780Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-09-27T23:04:13.797Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-09-27T23:04:13.797Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-09-27T23:04:13.813Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-09-27T23:04:13.813Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-09-27T23:04:13.830Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-09-27T23:04:13.830Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-09-27T23:04:13.849Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-09-27T23:04:13.849Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-09-27T23:04:13.867Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-09-27T23:04:13.867Zdownloading input: /input/build/work/scripts/test_up.sh
222024-09-27T23:04:13.887Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-09-27T23:04:13.887Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-09-27T23:04:13.942Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-09-27T23:04:13.944Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-09-27T23:04:14.709Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-09-27T23:04:14.709Zstarting task 0: "setup"
282024-09-27T23:04:14.719Z++ uname -s
292024-09-27T23:04:14.722Z+ kern=SunOS
302024-09-27T23:04:14.722Z+ case "$kern" in
312024-09-27T23:04:14.722Z+ groupadd -g 12345 build
322024-09-27T23:04:14.724Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-09-27T23:04:16.735Z+ zfs create -o mountpoint=/work rpool/work
342024-09-27T23:04:17.251Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-09-27T23:04:17.253Z+ home_fs=zfs
362024-09-27T23:04:17.253Z+ [[ zfs == autofs ]]
372024-09-27T23:04:17.253Z+ mkdir -p /home/build
382024-09-27T23:04:17.256Z+ chown build:build /home/build /work
392024-09-27T23:04:18.256Z+ chmod 0700 /home/build /work
402024-09-27T23:04:18.259Zprocess exited: duration 3550 ms, exit code 0
 
412024-09-27T23:04:18.265Zstarting task 1: "authentication"
422024-09-27T23:04:18.282Zprocess exited: duration 16 ms, exit code 0
 
432024-09-27T23:04:18.288Zstarting task 2: "build"
442024-09-27T23:04:18.294Z+ banner cores
452024-09-27T23:04:18.317Z
462024-09-27T23:04:18.317Z #### #### ##### ###### ####
472024-09-27T23:04:18.317Z # # # # # # # #
482024-09-27T23:04:18.317Z # # # # # ##### ####
492024-09-27T23:04:18.317Z # # # ##### # #
502024-09-27T23:04:18.317Z # # # # # # # # #
512024-09-27T23:04:18.317Z #### #### # # ###### ####
522024-09-27T23:04:18.317Z
532024-09-27T23:04:18.320Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-09-27T23:04:18.327Z+ echo 'input bins dir contains:'
552024-09-27T23:04:18.327Zinput bins dir contains:
562024-09-27T23:04:18.330Z+ ls -ltr /input/build/work/bins
572024-09-27T23:04:18.330Ztotal 771769
582024-09-27T23:04:18.330Z-rw-r--r-- 1 root root 110734273 Sep 27 23:04 crucible-downstairs.gz
592024-09-27T23:04:18.330Z-rw-r--r-- 1 root root 105947543 Sep 27 23:04 crucible-hammer.gz
602024-09-27T23:04:18.330Z-rw-r--r-- 1 root root 111407444 Sep 27 23:04 crutest.gz
612024-09-27T23:04:18.330Z-rw-r--r-- 1 root root 66799675 Sep 27 23:04 dsc.gz
622024-09-27T23:04:18.332Z+ banner unpack
632024-09-27T23:04:18.332Z
642024-09-27T23:04:18.332Z # # # # ##### ## #### # #
652024-09-27T23:04:18.332Z # # ## # # # # # # # # #
662024-09-27T23:04:18.333Z # # # # # # # # # # ####
672024-09-27T23:04:18.333Z # # # # # ##### ###### # # #
682024-09-27T23:04:18.333Z # # # ## # # # # # # #
692024-09-27T23:04:18.333Z #### # # # # # #### # #
702024-09-27T23:04:18.333Z
712024-09-27T23:04:18.333Z+ mkdir -p /var/tmp/bins
722024-09-27T23:04:18.335Z+ for t in "$input/bins/"*.gz
732024-09-27T23:04:18.335Z++ basename /input/build/work/bins/crucible-downstairs.gz
742024-09-27T23:04:18.338Z+ b=crucible-downstairs.gz
752024-09-27T23:04:18.338Z+ b=crucible-downstairs
762024-09-27T23:04:18.338Z+ gunzip
772024-09-27T23:04:20.634Z+ chmod +x /var/tmp/bins/crucible-downstairs
782024-09-27T23:04:20.636Z+ for t in "$input/bins/"*.gz
792024-09-27T23:04:20.640Z++ basename /input/build/work/bins/crucible-hammer.gz
802024-09-27T23:04:20.640Z+ b=crucible-hammer.gz
812024-09-27T23:04:20.640Z+ b=crucible-hammer
822024-09-27T23:04:20.640Z+ gunzip
832024-09-27T23:04:22.858Z+ chmod +x /var/tmp/bins/crucible-hammer
842024-09-27T23:04:22.861Z+ for t in "$input/bins/"*.gz
852024-09-27T23:04:22.863Z++ basename /input/build/work/bins/crutest.gz
862024-09-27T23:04:22.863Z+ b=crutest.gz
872024-09-27T23:04:22.863Z+ b=crutest
882024-09-27T23:04:22.863Z+ gunzip
892024-09-27T23:04:25.179Z+ chmod +x /var/tmp/bins/crutest
902024-09-27T23:04:25.181Z+ for t in "$input/bins/"*.gz
912024-09-27T23:04:25.184Z++ basename /input/build/work/bins/dsc.gz
922024-09-27T23:04:25.184Z+ b=dsc.gz
932024-09-27T23:04:25.184Z+ b=dsc
942024-09-27T23:04:25.184Z+ gunzip
952024-09-27T23:04:26.621Z+ chmod +x /var/tmp/bins/dsc
962024-09-27T23:04:26.624Z+ export BINDIR=/var/tmp/bins
972024-09-27T23:04:26.624Z+ BINDIR=/var/tmp/bins
982024-09-27T23:04:26.624Z+ jobpid=1139
992024-09-27T23:04:26.627Z+ echo 'Setup debug logging'
1002024-09-27T23:04:26.627ZSetup debug logging
1012024-09-27T23:04:26.627Z+ mkdir /tmp/debug
1022024-09-27T23:04:26.627Z+ sleep 7200
1032024-09-27T23:04:26.627Z+ psrinfo -v
1042024-09-27T23:04:26.630Z+ df -h
1052024-09-27T23:04:26.657Z+ prstat -d d -mLc 1
1062024-09-27T23:04:26.660Z+ iostat -T d -xn 1
1072024-09-27T23:04:26.660Z+ mpstat -T d 1
1082024-09-27T23:04:26.660Z+ vmstat -T d -p 1
1092024-09-27T23:04:26.663Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1102024-09-27T23:04:26.663Z+ banner test_up_encrypted
1112024-09-27T23:04:26.663Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1122024-09-27T23:04:26.663Z
1132024-09-27T23:04:26.663Z ##### ###### #### ##### # # ##### ###### # #
1142024-09-27T23:04:26.663Z # # # # # # # # # ## #
1152024-09-27T23:04:26.663Z # ##### #### # # # # # ##### # # #
1162024-09-27T23:04:26.663Z # # # # # # ##### # # # #
1172024-09-27T23:04:26.663Z # # # # # # # # # # ##
1182024-09-27T23:04:26.663Z # ###### #### # ####### #### # ####### ###### # #
1192024-09-27T23:04:26.663Z
1202024-09-27T23:04:26.663Z+ ptime -m bash /input/build/work/scripts/test_up.sh -N encrypted
1212024-09-27T23:04:26.675Z/input/build/work
1222024-09-27T23:04:26.707ZTurn off color for downstairs dump
1232024-09-27T23:04:26.745ZUpstairs using key: BDv0sX42iDSrvbYTu1XK3n4bo3vfP6dstEcQWsBUkSo=
1242024-09-27T23:04:26.748Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1252024-09-27T23:04:26.748ZCreating three downstairs regions
1262024-09-27T23:04:27.143ZStarting three downstairs
1272024-09-27T23:04:32.153ZDisable automatic restart on all downstairs
1282024-09-27T23:04:32.212Z
1292024-09-27T23:04:32.212ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1302024-09-27T23:04:32.215ZRunning test: span
1312024-09-27T23:04:36.364ZCompleted test: span
1322024-09-27T23:04:41.372ZRunning test: big
1332024-09-27T23:04:45.784ZCompleted test: big
1342024-09-27T23:04:50.792ZRunning test: dep
1352024-09-27T23:05:44.702ZCompleted test: dep
1362024-09-27T23:05:49.710ZRunning test: balloon
1372024-09-27T23:06:36.749ZCompleted test: balloon
1382024-09-27T23:06:41.765ZRunning test: deactivate
1392024-09-27T23:07:36.483ZCompleted test: deactivate
1402024-09-27T23:07:41.491ZRunning hammer
1412024-09-27T23:07:55.433ZRun repair tests
1422024-09-27T23:07:55.437Z/var/tmp/bins/crutest fill -g 31 -q --key BDv0sX42iDSrvbYTu1XK3n4bo3vfP6dstEcQWsBUkSo= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1432024-09-27T23:07:55.498Z{"msg":"use region info: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.540261637Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"task":"crutest"}
1442024-09-27T23:07:55.502Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.54090368Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"task":"crutest"}
1452024-09-27T23:07:55.502Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.542609789Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"task":"crutest"}
1462024-09-27T23:07:55.502Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.542816541Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219}
1472024-09-27T23:07:55.502Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"60f78286ad49cec90cdcc3e028e5e9c76d4fdc1d\",\n git_commit_timestamp: \"2024-09-27T22:46:32.000000000Z\",\n git_branch: \"alan/tests-use-dsc\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.542856616Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219}
1482024-09-27T23:07:55.502Z{"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.542877953Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219}
1492024-09-27T23:07:55.502Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.542893521Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219}
1502024-09-27T23:07:55.503Z{"msg":"Crucible 258d09bb-3bd2-424d-b6b6-0cc178035945 has session id: 68a5dc48-cfb0-4097-bc9c-ddfb1d20a528","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.543410341Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1512024-09-27T23:07:55.503Z{"msg":"Upstairs opts: Upstairs UUID: 258d09bb-3bd2-424d-b6b6-0cc178035945, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.54349269Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1522024-09-27T23:07:55.503Z{"msg":"Crucible stats registered with UUID: 258d09bb-3bd2-424d-b6b6-0cc178035945","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.543561051Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1532024-09-27T23:07:55.536Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.578134207Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219}
1542024-09-27T23:07:55.536Z{"msg":"258d09bb-3bd2-424d-b6b6-0cc178035945 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.57819055Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1552024-09-27T23:07:55.536Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.578214397Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"client":"0","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1562024-09-27T23:07:55.541Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.578251332Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"client":"0","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1572024-09-27T23:07:55.541Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.578299065Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"client":"1","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1582024-09-27T23:07:55.541Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.578319652Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"client":"1","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1592024-09-27T23:07:55.541Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.578369116Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"client":"2","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1602024-09-27T23:07:55.541Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.578402381Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"client":"2","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1612024-09-27T23:07:55.541Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.578452564Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"":"io task","client":"2","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1622024-09-27T23:07:55.541Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.578498358Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"":"io task","client":"0","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1632024-09-27T23:07:55.541Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.578540093Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"":"io task","client":"1","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1642024-09-27T23:07:55.542Z{"msg":"ds_connection connected from Ok(127.0.0.1:52380)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.578642379Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"":"io task","client":"0","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1652024-09-27T23:07:55.542Z{"msg":"ds_connection connected from Ok(127.0.0.1:34797)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.578741195Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"":"io task","client":"1","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1662024-09-27T23:07:55.542Z{"msg":"ds_connection connected from Ok(127.0.0.1:53689)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.578817655Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"":"io task","client":"2","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1672024-09-27T23:07:55.542Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.579344474Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"client":"2","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1682024-09-27T23:07:55.542Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.579381189Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"client":"2","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1692024-09-27T23:07:55.542Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.579460528Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"client":"1","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1702024-09-27T23:07:55.542Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.57952076Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"client":"1","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1712024-09-27T23:07:55.542Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.579537938Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"client":"0","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1722024-09-27T23:07:55.542Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.579553016Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"client":"0","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1732024-09-27T23:07:55.542Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.580051368Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"client":"1","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1742024-09-27T23:07:55.542Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.580103991Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"client":"0","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1752024-09-27T23:07:55.542Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.580161323Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"client":"2","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1762024-09-27T23:07:55.543Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.580501567Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"client":"1","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1772024-09-27T23:07:55.543Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.580537763Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"client":"1","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1782024-09-27T23:07:55.543Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.58055565Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1792024-09-27T23:07:55.543Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.580586316Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"client":"2","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1802024-09-27T23:07:55.543Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.580616902Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"client":"2","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1812024-09-27T23:07:55.543Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.580651207Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1822024-09-27T23:07:55.543Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.580665965Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"client":"0","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1832024-09-27T23:07:55.543Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.580684483Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"client":"0","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1842024-09-27T23:07:55.543Z{"msg":"[0]R flush_numbers: [1689, 1689, 1689, 1689, 1689]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.580714499Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"":"downstairs","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1852024-09-27T23:07:55.543Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
1862024-09-27T23:07:55.543Z,"time":"2024-09-27T23:07:54.580729986Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"":"downstairs","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1872024-09-27T23:07:55.543Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.580765282Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"":"downstairs","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1882024-09-27T23:07:55.543Z{"msg":"[1]R flush_numbers: [1689, 1689, 1689, 1689, 1689]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.580797867Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"":"downstairs","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1892024-09-27T23:07:55.544Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.580829423Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"":"downstairs","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1902024-09-27T23:07:55.544Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.58085007Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"":"downstairs","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1912024-09-27T23:07:55.544Z{"msg":"[2]R flush_numbers: [1689, 1689, 1689, 1689, 1689]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.580879346Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"":"downstairs","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1922024-09-27T23:07:55.544Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.580896284Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"":"downstairs","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1932024-09-27T23:07:55.544Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.58092682Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"":"downstairs","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1942024-09-27T23:07:55.544Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.580941828Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"":"downstairs","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1952024-09-27T23:07:55.544Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.580973544Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"":"downstairs","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1962024-09-27T23:07:55.544ZRegion: es:10 ec:5 bs:4096 ts:204800 tb:50 max_io:50 or 204800
1972024-09-27T23:07:55.544Z{"msg":"Next flush: 1690","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.581010788Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"":"downstairs","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1982024-09-27T23:07:55.544Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.581027346Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"":"downstairs","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
1992024-09-27T23:07:55.544Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.581041415Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"session_id":"Fill test
2002024-09-27T23:07:55.544Z68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
2012024-09-27T23:07:55.544Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.581056062Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"client":"0","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
2022024-09-27T23:07:55.544Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.581069611Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"client":"1","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
2032024-09-27T23:07:55.544Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.581091098Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"client":"2","":"downstairs","session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
2042024-09-27T23:07:55.545Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.581105116Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
2052024-09-27T23:07:55.545Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.581118384Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
2062024-09-27T23:07:55.545Z{"msg":"258d09bb-3bd2-424d-b6b6-0cc178035945 is now active with session: 68a5dc48-cfb0-4097-bc9c-ddfb1d20a528","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.581131632Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
2072024-09-27T23:07:55.545Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.58115092Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219,"session_id":"68a5dc48-cfb0-4097-bc9c-ddfb1d20a528"}
2082024-09-27T23:07:55.545Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:54.581163558Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219}
2092024-09-27T23:07:55.588ZRead and Verify all blocks (0..50 range:false)
2102024-09-27T23:07:55.632ZWrote out file "/var/tmp/test_up-build/verify_file"
2112024-09-27T23:07:55.632ZCLIENT: Tests done. All submitted work has been ACK'd
2122024-09-27T23:07:55.635Z----------------------------------------------------------------
2132024-09-27T23:07:55.635Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2142024-09-27T23:07:55.635ZStates: Active Active Active
2152024-09-27T23:07:55.635ZGW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2162024-09-27T23:07:55.635Z 3 Acked 1002 Read 50 Sent Done Sent false
2172024-09-27T23:07:55.635Z STATES DS:0 DS:1 DS:2 TOTAL
2182024-09-27T23:07:55.635Z Sent 1 0 1 2
2192024-09-27T23:07:55.635Z Done 0 1 0 1
2202024-09-27T23:07:55.636Z Skipped 0 0 0 0
2212024-09-27T23:07:55.636Z Error 0 0 0 0
2222024-09-27T23:07:55.636ZLast Flush: 1001 1001 1001
2232024-09-27T23:07:55.636ZDownstairs last five completed: 1001 1000
2242024-09-27T23:07:55.636ZUpstairs last five completed: 3 2 1
2252024-09-27T23:07:55.636ZCLIENT: Up:0 ds:1 act:3
2262024-09-27T23:07:59.633Z----------------------------------------------------------------
2272024-09-27T23:07:59.633Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2282024-09-27T23:07:59.633ZDownstairs last five completed: 1003 1002 1001 1000
2292024-09-27T23:07:59.633ZUpstairs last five completed: 4 3 2 1
2302024-09-27T23:07:59.633ZCLIENT: Up:0 ds:0 act:3
2312024-09-27T23:07:59.633ZCLIENT: All crucible jobs finished, exiting program
2322024-09-27T23:07:59.637Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T23:07:58.674794164Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1219}
2332024-09-27T23:07:59.644ZRepair setup passed
2342024-09-27T23:07:59.648ZCopy the region for /var/tmp/test_up-build/8830
2352024-09-27T23:07:59.650Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2362024-09-27T23:07:59.653Z/var/tmp/bins/crutest repair -g 32 -q --key BDv0sX42iDSrvbYTu1XK3n4bo3vfP6dstEcQWsBUkSo= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2372024-09-27T23:07:59.713Z{"msg":"use region info: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.753814813Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"task":"crutest"}
2382024-09-27T23:07:59.716Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.754344781Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"task":"crutest"}
2392024-09-27T23:07:59.716Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.756020264Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"task":"crutest"}
2402024-09-27T23:07:59.716Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.756212289Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225}
2412024-09-27T23:07:59.716Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"60f78286ad49cec90cdcc3e028e5e9c76d4fdc1d\",\n git_commit_timestamp: \"2024-09-27T22:46:32.000000000Z\",\n git_branch: \"alan/tests-use-dsc\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.756259362Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225}
2422024-09-27T23:07:59.716Z{"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.756286279Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225}
2432024-09-27T23:07:59.716Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.756310665Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225}
2442024-09-27T23:07:59.716Z{"msg":"Crucible 8a55a8ba-72f3-4f7b-bda4-86dae58458e4 has session id: d213bef1-2035-47f8-bfde-b7e30a7cce22","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.756750626Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2452024-09-27T23:07:59.716Z{"msg":"Upstairs opts: Upstairs UUID: 8a55a8ba-72f3-4f7b-bda4-86dae58458e4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.756799099Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2462024-09-27T23:07:59.716Z{"msg":"Crucible stats registered with UUID: 8a55a8ba-72f3-4f7b-bda4-86dae58458e4","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.756837054Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2472024-09-27T23:07:59.752Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.791159435Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225}
2482024-09-27T23:07:59.752Z{"msg":"8a55a8ba-72f3-4f7b-bda4-86dae58458e4 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.791208978Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2492024-09-27T23:07:59.752Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.791229425Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"client":"0","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2502024-09-27T23:07:59.756Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.79126937Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"client":"0","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2512024-09-27T23:07:59.756Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.791306625Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"client":"1","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2522024-09-27T23:07:59.756Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.791328922Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"client":"1","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2532024-09-27T23:07:59.756Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.791356818Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"client":"2","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2542024-09-27T23:07:59.756Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.791403212Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"client":"2","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2552024-09-27T23:07:59.756Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.791431688Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"":"io task","client":"2","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2562024-09-27T23:07:59.757Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.791460454Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"":"io task","client":"1","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2572024-09-27T23:07:59.757Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.791516586Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"":"io task","client":"0","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2582024-09-27T23:07:59.757Z{"msg":"ds_connection connected from Ok(127.0.0.1:65150)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.791549022Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"":"io task","client":"2","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2592024-09-27T23:07:59.757Z{"msg":"ds_connection connected from Ok(127.0.0.1:45489)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.791591047Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"":"io task","client":"0","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2602024-09-27T23:07:59.757Z{"msg":"ds_connection connected from Ok(127.0.0.1:54194)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.791822055Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"":"io task","client":"1","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2612024-09-27T23:07:59.757Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.792369411Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"client":"2","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2622024-09-27T23:07:59.757Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.792413655Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"client":"2","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2632024-09-27T23:07:59.757Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.792443101Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"client":"0","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2642024-09-27T23:07:59.757Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.792492854Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"client":"0","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2652024-09-27T23:07:59.757Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.792552237Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"client":"1","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2662024-09-27T23:07:59.757Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.792587682Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"client":"1","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2672024-09-27T23:07:59.757Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.793148626Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"client":"0","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2682024-09-27T23:07:59.757Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.793183311Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"client":"1","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2692024-09-27T23:07:59.757Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.793200519Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"client":"2","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2702024-09-27T23:07:59.757Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.793554261Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"client":"2","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2712024-09-27T23:07:59.757Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.793595136Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"client":"2","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2722024-09-27T23:07:59.757Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.793612003Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2732024-09-27T23:07:59.757Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.793725288Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"client":"1","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2742024-09-27T23:07:59.757Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.793757364Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"client":"1","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2752024-09-27T23:07:59.757Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.793771572Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2762024-09-27T23:07:59.757Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.79378502Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"client":"0","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2772024-09-27T23:07:59.757Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.793806017Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"client":"0","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2782024-09-27T23:07:59.757Z{"msg":"[0]R flush_numbers: [1690, 1690, 1690, 1690, 1690]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.793822965Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"":"downstairs","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2792024-09-27T23:07:59.757Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.793843682Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"":"downstairs","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2802024-09-27T23:07:59.757Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.79385755Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"":"downstairs","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2812024-09-27T23:07:59.757Z{"msg":"[1]R flush_numbers: [1690, 1690, 1690, 1690, 1690]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.793878697Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"":"downstairs","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2822024-09-27T23:07:59.757Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.793892215Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"":"downstairs","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2832024-09-27T23:07:59.757Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.793912163Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"":"downstairs","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2842024-09-27T23:07:59.757Z{"msg":"[2]R flush_numbers: [1690, 1690, 1690, 1690, 1690]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.793925921Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"":"downstairs","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2852024-09-27T23:07:59.757Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.793946428Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"":"downstairs","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2862024-09-27T23:07:59.757Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.793961516Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"":"downstairs","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2872024-09-27T23:07:59.757Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.793974864Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"":"downstairs","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2882024-09-27T23:07:59.757Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.793988182Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"":"downstairs","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2892024-09-27T23:07:59.757Z{"msg":"Next flush: 1691","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
2902024-09-27T23:07:59.757Z,"time":"2024-09-27T23:07:58.79400819Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"":"downstairs","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2912024-09-27T23:07:59.757Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.794028977Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"":"downstairs","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2922024-09-27T23:07:59.757Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.794049234Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2932024-09-27T23:07:59.757Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.794061542Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"client":"0","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2942024-09-27T23:07:59.758Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.794074771Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"client":"1","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2952024-09-27T23:07:59.758Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.794094608Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"client":"2","":"downstairs","session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2962024-09-27T23:07:59.758Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.794108216Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2972024-09-27T23:07:59.758Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.794127224Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2982024-09-27T23:07:59.758Z{"msg":"8a55a8ba-72f3-4f7b-bda4-86dae58458e4 is now active with session: d213bef1-2035-47f8-bfde-b7e30a7cce22","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.794140672Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
2992024-09-27T23:07:59.758Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.794159299Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225,"session_id":"d213bef1-2035-47f8-bfde-b7e30a7cce22"}
3002024-09-27T23:07:59.758Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:58.794171818Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225}
3012024-09-27T23:07:59.758ZRegion: es:10 ec:5 bs:4096 ts:204800 tb:50 max_io:50 or 204800
3022024-09-27T23:07:59.758ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
3032024-09-27T23:07:59.758ZRun Repair workload
3042024-09-27T23:07:59.758Z01/10 Write block 32 len 24576 data: 2 2 2 2 2 2
3052024-09-27T23:07:59.758Z02/10 Read block 36 len 20480
3062024-09-27T23:07:59.765Z03/10 Write block 17 len 16384 data: 2 2 2 2
3072024-09-27T23:07:59.768Z04/10 Write block 11 len 36864 data: 2 2 2 2 2 2 3 3 3
3082024-09-27T23:07:59.768Z05/10 Write block 8 len 28672 data: 2 2 2 3 3 3 3
3092024-09-27T23:07:59.768Z06/10 Read block 24 len 32768
3102024-09-27T23:07:59.782Z07/10 Read block 37 len 16384
3112024-09-27T23:07:59.787Z08/10 Read block 11 len 20480
3122024-09-27T23:07:59.794Z09/10 Read block 28 len 40960
3132024-09-27T23:07:59.802Z10/10 Write block 3 len 4096 data: 2
3142024-09-27T23:07:59.805Z----------------------------------------------------------------
3152024-09-27T23:07:59.805Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:10
3162024-09-27T23:07:59.805ZStates: Active Active Active
3172024-09-27T23:07:59.805ZGW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
3182024-09-27T23:07:59.805Z 1 Acked 1000 Write 6 Done Done Done false
3192024-09-27T23:07:59.805Z 2 Acked 1001 Read 5 Done Done Done false
3202024-09-27T23:07:59.806Z 3 Acked 1002 Write 4 Done Done Done false
3212024-09-27T23:07:59.806Z 4 Acked 1003 Write 9 Done Done Done false
3222024-09-27T23:07:59.806Z 5 Acked 1004 Write 7 Done Done Done false
3232024-09-27T23:07:59.806Z 6 Acked 1005 Read 8 Done Done Done false
3242024-09-27T23:07:59.806Z 7 Acked 1006 Read 4 Done Done Done false
3252024-09-27T23:07:59.806Z 8 Acked 1007 Read 5 Done Done Done false
3262024-09-27T23:07:59.806Z 9 Acked 1008 Read 10 Done Done Done false
3272024-09-27T23:07:59.806Z 10 Acked 1009 Write 1 Sent Sent Sent false
3282024-09-27T23:07:59.806Z STATES DS:0 DS:1 DS:2 TOTAL
3292024-09-27T23:07:59.806Z Sent 1 1 1 3
3302024-09-27T23:07:59.806Z Done 9 9 9 27
3312024-09-27T23:07:59.806Z Skipped 0 0 0 0
3322024-09-27T23:07:59.806Z Error 0 0 0 0
3332024-09-27T23:07:59.806ZLast Flush: 0 0 0
3342024-09-27T23:07:59.806ZDownstairs last five completed:
3352024-09-27T23:07:59.806ZUpstairs last five completed: 10 9 8 7 6
3362024-09-27T23:07:59.806Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T23:07:58.844341786Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1225}
3372024-09-27T23:07:59.806ZWrote out file "/var/tmp/test_up-build/verify_file"
3382024-09-27T23:07:59.810ZRepair part 1 passed
3392024-09-27T23:07:59.813Z
3402024-09-27T23:07:59.813ZKill the current downstairs
3412024-09-27T23:07:59.925Zrm -rf /var/tmp/test_up-build/8830
3422024-09-27T23:07:59.925ZNow put back the original so we have a mismatch
3432024-09-27T23:07:59.925Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
3442024-09-27T23:07:59.930ZRestart downstairs with old directory
3452024-09-27T23:07:59.987Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
3462024-09-27T23:08:00.006ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3472024-09-27T23:08:00.009Z 0 000-009 31 31 31 1690 1690 1690 T T F <---
3482024-09-27T23:08:00.009Z 1 010-019 31 31 31 1690 1690 1690 T T F <---
3492024-09-27T23:08:00.009Z 2 020-029 31 31 31 1690 1690 1690 T T F <---
3502024-09-27T23:08:00.009Z 3 030-039 31 31 31 1690 1690 1690 T T F <---
3512024-09-27T23:08:00.009Z 4 040-049 31 31 31 1690 1690 1690 F F F
3522024-09-27T23:08:00.009ZMax gen: 31, Max flush: 1690
3532024-09-27T23:08:00.009ZError: Difference in extent metadata found!
3542024-09-27T23:08:00.012Zdump test found error as expected
3552024-09-27T23:08:00.013Z
3562024-09-27T23:08:00.013Z
3572024-09-27T23:08:00.013Z/var/tmp/bins/crutest verify --range -g 33 -q --key BDv0sX42iDSrvbYTu1XK3n4bo3vfP6dstEcQWsBUkSo= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
3582024-09-27T23:08:00.069Z{"msg":"use region info: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.110409601Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"task":"crutest"}
3592024-09-27T23:08:00.073Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.110978015Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"task":"crutest"}
3602024-09-27T23:08:00.073Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.112685284Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"task":"crutest"}
3612024-09-27T23:08:00.073Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.112903304Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232}
3622024-09-27T23:08:00.073Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"60f78286ad49cec90cdcc3e028e5e9c76d4fdc1d\",\n git_commit_timestamp: \"2024-09-27T22:46:32.000000000Z\",\n git_branch: \"alan/tests-use-dsc\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.112942359Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232}
3632024-09-27T23:08:00.073Z{"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.112964066Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232}
3642024-09-27T23:08:00.073Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.112979624Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232}
3652024-09-27T23:08:00.073Z{"msg":"Crucible 49af88cd-4534-46a3-8da4-3f03c0825e79 has session id: 12017c72-ccba-44da-b096-de2f1bf7d16f","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.113652253Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
3662024-09-27T23:08:00.073Z{"msg":"Upstairs opts: Upstairs UUID: 49af88cd-4534-46a3-8da4-3f03c0825e79, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.113708786Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
3672024-09-27T23:08:00.073Z{"msg":"Crucible stats registered with UUID: 49af88cd-4534-46a3-8da4-3f03c0825e79","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.113742651Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
3682024-09-27T23:08:00.108Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.148477716Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232}
3692024-09-27T23:08:00.108Z{"msg":"49af88cd-4534-46a3-8da4-3f03c0825e79 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.148548666Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
3702024-09-27T23:08:00.108Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.148584601Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"0","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
3712024-09-27T23:08:00.112Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.148632395Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"0","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
3722024-09-27T23:08:00.112Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.148687647Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"1","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
3732024-09-27T23:08:00.112Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.14874156Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"1","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
3742024-09-27T23:08:00.112Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.148778405Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"2","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
3752024-09-27T23:08:00.112Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.148826719Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"2","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
3762024-09-27T23:08:00.112Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.148859014Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
3772024-09-27T23:08:00.112Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.148903608Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
3782024-09-27T23:08:00.112Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.148953611Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
3792024-09-27T23:08:00.112Z{"msg":"ds_connection connected from Ok(127.0.0.1:36191)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.149002975Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
3802024-09-27T23:08:00.112Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-09-27T23:07:59.149041239Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
3812024-09-27T23:08:00.112Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-09-27T23:07:59.149077665Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
3822024-09-27T23:08:00.112Z{"msg":"ds_connection connected from Ok(127.0.0.1:57623)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.14911188Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
3832024-09-27T23:08:00.112Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-09-27T23:07:59.149161283Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
3842024-09-27T23:08:00.112Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.149210776Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"2","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
3852024-09-27T23:08:00.112Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.149243222Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"2","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
3862024-09-27T23:08:00.113Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.149269599Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
3872024-09-27T23:08:00.113Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.149529633Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"0","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
3882024-09-27T23:08:00.113Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.149584166Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"0","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
3892024-09-27T23:08:00.113Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.149676893Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"1","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
3902024-09-27T23:08:00.113Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.149725387Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"1","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
3912024-09-27T23:08:00.113Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.150235728Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"0","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
3922024-09-27T23:08:00.113Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.150280102Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"1","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
3932024-09-27T23:08:00.113Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.150605528Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"1","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
3942024-09-27T23:08:00.113Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.150642413Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"1","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
3952024-09-27T23:08:00.113Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.150658201Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
3962024-09-27T23:08:00.113Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.150678468Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"0","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
3972024-09-27T23:08:00.113Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.150692426Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"0","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
3982024-09-27T23:08:00.113Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:07:59.150712384Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
3992024-09-27T23:08:10.110Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.149747007Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4002024-09-27T23:08:10.110Z{"msg":"ds_connection connected from Ok(127.0.0.1:40242)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.149916524Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4012024-09-27T23:08:10.113Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.150961833Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"2","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4022024-09-27T23:08:10.113Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.151002117Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"2","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4032024-09-27T23:08:10.113Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.151568441Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"2","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4042024-09-27T23:08:10.113Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.151942441Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"2","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4052024-09-27T23:08:10.113Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.151977646Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"2","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4062024-09-27T23:08:10.113Z{"msg":"[0]R flush_numbers: [1690, 1690, 1690, 1690, 1690]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.151998353Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4072024-09-27T23:08:10.113Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152015871Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4082024-09-27T23:08:10.113Z{"msg":"[0]R dirty: [true, true, true, true, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152042597Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4092024-09-27T23:08:10.113Z{"msg":"[1]R flush_numbers: [1690, 1690, 1690, 1690, 1690]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152059854Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4102024-09-27T23:08:10.113Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152076572Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4112024-09-27T23:08:10.113Z{"msg":"[1]R dirty: [true, true, true, true, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152101699Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4122024-09-27T23:08:10.113Z{"msg":"[2]R flush_numbers: [1690, 1690, 1690, 1690, 1690]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152119046Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4132024-09-27T23:08:10.113Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152144403Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4142024-09-27T23:08:10.113Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152161341Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4152024-09-27T23:08:10.113Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152185797Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4162024-09-27T23:08:10.113Z{"msg":"Generation requested: 33 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152202615Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4172024-09-27T23:08:10.114Z{"msg":"Next flush: 1691","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152226922Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4182024-09-27T23:08:10.114Z{"msg":"Extents 0 dirty","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.15224348Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4192024-09-27T23:08:10.114Z{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152269106Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4202024-09-27T23:08:10.114Z{"msg":"extent:0 gens: 31 31 31","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152288224Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4212024-09-27T23:08:10.114Z{"msg":"extent:0 flush: 1690 1690 1690 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152309881Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4222024-09-27T23:08:10.114Z{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152324469Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4232024-09-27T23:08:10.114Z{"msg":"extent:0 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152338707Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4242024-09-27T23:08:10.114Z{"msg":"find dest for source 0 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152353115Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4252024-09-27T23:08:10.114Z{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152367223Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4262024-09-27T23:08:10.114Z{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.15238814Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4272024-09-27T23:08:10.114Z{"msg":"Extents 1 dirty","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152409007Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4282024-09-27T23:08:10.114Z{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152431164Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4292024-09-27T23:08:10.114Z{"msg":"extent:1 gens: 31 31 31","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152445582Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4302024-09-27T23:08:10.114Z{"msg":"extent:1 flush: 1690 1690 1690 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.15246118Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4312024-09-27T23:08:10.114Z{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152475768Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4322024-09-27T23:08:10.114Z{"msg":"extent:1 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152489907Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4332024-09-27T23:08:10.114Z{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152504254Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4342024-09-27T23:08:10.117Z{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152518992Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4352024-09-27T23:08:10.117Z{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.15253397Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4362024-09-27T23:08:10.117Z{"msg":"Extents 2 dirty","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152548299Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4372024-09-27T23:08:10.118Z{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152561867Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4382024-09-27T23:08:10.118Z{"msg":"extent:2 gens: 31 31 31","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152583344Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4392024-09-27T23:08:10.118Z{"msg":"extent:2 flush: 1690 1690 1690 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152598502Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4402024-09-27T23:08:10.118Z{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.15261312Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4412024-09-27T23:08:10.118Z{"msg":"extent:2 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152628128Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4422024-09-27T23:08:10.118Z{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152643236Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4432024-09-27T23:08:10.118Z{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152679551Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4442024-09-27T23:08:10.118Z{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152711717Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4452024-09-27T23:08:10.118Z{"msg":"Extents 3 dirty","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.15276046Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4462024-09-27T23:08:10.118Z{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152797955Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4472024-09-27T23:08:10.118Z{"msg":"extent:3 gens: 31 31 31","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.15283252Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4482024-09-27T23:08:10.118Z{"msg":"extent:3 flush: 1690 1690 1690 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152865426Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4492024-09-27T23:08:10.118Z{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152881683Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4502024-09-27T23:08:10.118Z{"msg":"extent:3 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152897411Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4512024-09-27T23:08:10.118Z{"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152931457Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4522024-09-27T23:08:10.118Z{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.152964762Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4532024-09-27T23:08:10.118Z{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.15298166Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4542024-09-27T23:08:10.118Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.153003847Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"0","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4552024-09-27T23:08:10.118Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.153018055Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"1","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4562024-09-27T23:08:10.118Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.153038452Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"2","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4572024-09-27T23:08:10.118Z{"msg":"starting reconciliation f02bf334-0b29-4ea3-b11a-3c5595cebdf4: found 4 extents that need repair","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.15305273Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4582024-09-27T23:08:10.118Z{"msg":"Full repair list: {ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.153077657Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4592024-09-27T23:08:10.118Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 1691, gen_number: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 1691, gen_number: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 1691, gen_number: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 1691, gen_number: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.153158296Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4602024-09-27T23:08:10.118Z{"msg":"no Nexus client from DNS, aborting start notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:08:09.153181893Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"reconcile":"f02bf334-0b29-4ea3-b11a-3c5595cebdf4","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4612024-09-27T23:08:10.118Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:08:09.153197431Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"reconcile":"f02bf334-0b29-4ea3-b11a-3c5595cebdf4","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4622024-09-27T23:08:10.118Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:08:09.154278455Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"reconcile":"f02bf334-0b29-4ea3-b11a-3c5595cebdf4","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4632024-09-27T23:08:10.118Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.154566216Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"1","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4642024-09-27T23:08:10.118Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.154620668Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"2","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4652024-09-27T23:08:10.118Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:08:09.154667082Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"reconcile":"f02bf334-0b29-4ea3-b11a-3c5595cebdf4","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4662024-09-27T23:08:10.129Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:08:09.169237063Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"reconcile":"f02bf334-0b29-4ea3-b11a-3c5595cebdf4","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4672024-09-27T23:08:10.131Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:08:09.169676313Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"reconcile":"f02bf334-0b29-4ea3-b11a-3c5595cebdf4","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4682024-09-27T23:08:10.131Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:08:09.171293865Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"reconcile":"f02bf334-0b29-4ea3-b11a-3c5595cebdf4","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4692024-09-27T23:08:10.134Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.171632169Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"1","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4702024-09-27T23:08:10.134Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.17169761Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"2","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4712024-09-27T23:08:10.134Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:08:09.171739994Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"reconcile":"f02bf334-0b29-4ea3-b11a-3c5595cebdf4","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4722024-09-27T23:08:10.141Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:08:09.180998683Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"reconcile":"f02bf334-0b29-4ea3-b11a-3c5595cebdf4","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4732024-09-27T23:08:10.143Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:08:09.181399679Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"reconcile":"f02bf334-0b29-4ea3-b11a-3c5595cebdf4","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4742024-09-27T23:08:10.147Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:08:09.183601671Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"reconcile":"f02bf334-0b29-4ea3-b11a-3c5595cebdf4","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4752024-09-27T23:08:10.147Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.183920188Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"1","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4762024-09-27T23:08:10.147Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.183952574Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"2","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4772024-09-27T23:08:10.147Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:08:09.18405174Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"reconcile":"f02bf334-0b29-4ea3-b11a-3c5595cebdf4","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4782024-09-27T23:08:10.154Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:08:09.194433167Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"reconcile":"f02bf334-0b29-4ea3-b11a-3c5595cebdf4","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4792024-09-27T23:08:10.157Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:08:09.194850421Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"reconcile":"f02bf334-0b29-4ea3-b11a-3c5595cebdf4","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4802024-09-27T23:08:10.157Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:08:09.196460963Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"reconcile":"f02bf334-0b29-4ea3-b11a-3c5595cebdf4","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4812024-09-27T23:08:10.157Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.196755023Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"1","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4822024-09-27T23:08:10.157Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.196794358Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"2","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4832024-09-27T23:08:10.157Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:08:09.196811825Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"reconcile":"f02bf334-0b29-4ea3-b11a-3c5595cebdf4","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4842024-09-27T23:08:10.168Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:08:09.20602447Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"reconcile":"f02bf334-0b29-4ea3-b11a-3c5595cebdf4","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4852024-09-27T23:08:10.170Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.206413147Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4862024-09-27T23:08:10.171Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.206450733Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"0","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4872024-09-27T23:08:10.171Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.20646735Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"1","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4882024-09-27T23:08:10.171Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.206488947Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"client":"2","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4892024-09-27T23:08:10.171Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.206503046Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4902024-09-27T23:08:10.171Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.206522563Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4912024-09-27T23:08:10.171Z{"msg":"49af88cd-4534-46a3-8da4-3f03c0825e79 is now active with session: 12017c72-ccba-44da-b096-de2f1bf7d16f","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.206536311Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4922024-09-27T23:08:10.171Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2024-09-27T23:08:09.206562657Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232}
4932024-09-27T23:08:10.171ZWait for a query_work_queue command to finish before sending IO
4942024-09-27T23:08:10.171Z{"msg":"no Nexus client from DNS, aborting finish notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T23:08:09.206699269Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232,"reconcile":"f02bf334-0b29-4ea3-b11a-3c5595cebdf4","":"downstairs","":"downstairs","session_id":"12017c72-ccba-44da-b096-de2f1bf7d16f"}
4952024-09-27T23:08:10.171ZRegion: es:10 ec:5 bs:4096 ts:204800 tb:50 max_io:50 or 204800
4962024-09-27T23:08:10.171ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4972024-09-27T23:08:10.171ZRead and Verify all blocks (0..50 range:true)
4982024-09-27T23:08:10.208ZWrote out file "/var/tmp/test_up-build/verify_file"
4992024-09-27T23:08:10.209ZVerify test completed
5002024-09-27T23:08:10.211ZWrote out file "/var/tmp/test_up-build/verify_file"
5012024-09-27T23:08:10.211ZCLIENT: Tests done. All submitted work has been ACK'd
5022024-09-27T23:08:10.211Z----------------------------------------------------------------
5032024-09-27T23:08:10.211Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
5042024-09-27T23:08:10.211ZStates: Active Active Active
5052024-09-27T23:08:10.211ZGW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
5062024-09-27T23:08:10.211Z 1 Acked 1000 Read 50 Sent Done Sent false
5072024-09-27T23:08:10.211Z STATES DS:0 DS:1 DS:2 TOTAL
5082024-09-27T23:08:10.211Z Sent 1 0 1 2
5092024-09-27T23:08:10.211Z Done 0 1 0 1
5102024-09-27T23:08:10.211Z Skipped 0 0 0 0
5112024-09-27T23:08:10.211Z Error 0 0 0 0
5122024-09-27T23:08:10.211ZLast Flush: 0 0 0
5132024-09-27T23:08:10.211ZDownstairs last five completed:
5142024-09-27T23:08:10.211ZUpstairs last five completed: 1
5152024-09-27T23:08:10.211ZCLIENT: Up:0 ds:1 act:3
5162024-09-27T23:08:14.210Z----------------------------------------------------------------
5172024-09-27T23:08:14.211Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
5182024-09-27T23:08:14.211ZDownstairs last five completed: 1001 1000
5192024-09-27T23:08:14.211ZUpstairs last five completed: 2 1
5202024-09-27T23:08:14.211ZCLIENT: Up:0 ds:0 act:3
5212024-09-27T23:08:14.211ZCLIENT: All crucible jobs finished, exiting program
5222024-09-27T23:08:14.213Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T23:08:13.250118403Z","hostname":"w-01J8TVJQ3EWQ3ZCSF08GFSP7EF","pid":1232}
5232024-09-27T23:08:14.218ZRepair part 2 passed
5242024-09-27T23:08:14.220Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
5252024-09-27T23:08:14.235ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
5262024-09-27T23:08:14.235Z 0 000-009 32 32 32 1691 1691 1691 F F F
5272024-09-27T23:08:14.235Z 1 010-019 32 32 32 1691 1691 1691 F F F
5282024-09-27T23:08:14.235Z 2 020-029 32 32 32 1691 1691 1691 F F F
5292024-09-27T23:08:14.235Z 3 030-039 32 32 32 1691 1691 1691 F F F
5302024-09-27T23:08:14.235Z 4 040-049 31 31 31 1690 1690 1690 F F F
5312024-09-27T23:08:14.235ZMax gen: 32, Max flush: 1691
5322024-09-27T23:08:14.239Zdump test passed
5332024-09-27T23:08:14.242Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
5342024-09-27T23:08:14.256Z Extent 1
5352024-09-27T23:08:14.256ZGEN 32 32 32
5362024-09-27T23:08:14.256ZFLUSH_ID 1691 1691 1691
5372024-09-27T23:08:14.256ZDIRTY
5382024-09-27T23:08:14.256Z
5392024-09-27T23:08:14.256ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
5402024-09-27T23:08:14.258Z010 A A A A A A
5412024-09-27T23:08:14.258Z011 A A A A A A
5422024-09-27T23:08:14.261Z012 A A A A A A
5432024-09-27T23:08:14.261Z013 A A A A A A
5442024-09-27T23:08:14.264Z014 A A A A A A
5452024-09-27T23:08:14.266Z015 A A A A A A
5462024-09-27T23:08:14.266Z016 A A A A A A
5472024-09-27T23:08:14.266Z017 A A A A A A
5482024-09-27T23:08:14.269Z018 A A A A A A
5492024-09-27T23:08:14.269Z019 A A A A A A
5502024-09-27T23:08:14.273Zdump extent test passed
5512024-09-27T23:08:14.275Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
5522024-09-27T23:08:14.289ZExtent 2 Block in extent 0 Actual block 20
5532024-09-27T23:08:14.289Z
5542024-09-27T23:08:14.292Z DATA SHA256 VER
5552024-09-27T23:08:14.292Z------ ---------------------------------------------------------------- ---
5562024-09-27T23:08:14.292Z 0 97be19f400a29409ac5fafef1c6d8f4b3617ccb906e69145aec86a9dd3d53210 A
5572024-09-27T23:08:14.292Z 1 97be19f400a29409ac5fafef1c6d8f4b3617ccb906e69145aec86a9dd3d53210 A
5582024-09-27T23:08:14.292Z 2 97be19f400a29409ac5fafef1c6d8f4b3617ccb906e69145aec86a9dd3d53210 A
5592024-09-27T23:08:14.292Z
5602024-09-27T23:08:14.292ZNONCES 0 1 2 DIFF
5612024-09-27T23:08:14.292Z------ ------------------------ ------------------------ ------------------------ -----
5622024-09-27T23:08:14.294Z 0 94f500515d7c274c8c1c3aa2 94f500515d7c274c8c1c3aa2 94f500515d7c274c8c1c3aa2
5632024-09-27T23:08:14.294Z
5642024-09-27T23:08:14.294Z TAGS 0 1 2 DIFF
5652024-09-27T23:08:14.294Z------ -------------------------------- -------------------------------- -------------------------------- -----
5662024-09-27T23:08:14.294Z 0 0d0832f4e1a40044cc59eec7e2e86ce4 0d0832f4e1a40044cc59eec7e2e86ce4 0d0832f4e1a40044cc59eec7e2e86ce4
5672024-09-27T23:08:14.295Z
5682024-09-27T23:08:14.295ZHASHES 0 1 2 DIFF
5692024-09-27T23:08:14.295Z------ ---------------- ---------------- ---------------- -----
5702024-09-27T23:08:14.295Z
5712024-09-27T23:08:14.298Zdump block test passed
5722024-09-27T23:08:14.298ZUpstairs tests have completed, stopping all downstairs
5732024-09-27T23:08:19.361ZCreating four larger downstairs regions
5742024-09-27T23:08:19.913ZStarting four downstairs
5752024-09-27T23:08:24.921ZNow do the replace-reconcile test
5762024-09-27T23:11:20.246ZCompleted test: replace-reconcile
5772024-09-27T23:11:20.248ZNow do the replace-before-active test
5782024-09-27T23:14:01.073ZCompleted test: replace-before-active
5792024-09-27T23:14:01.076ZTests have completed, stopping all downstairs
5802024-09-27T23:14:01.134Z
5812024-09-27T23:14:01.134ZAll Tests have passed
5822024-09-27T23:14:01.137Z9:35 Test duration
5832024-09-27T23:14:01.137Z
5842024-09-27T23:14:01.137Zreal 9:34.382073013
5852024-09-27T23:14:01.137Zuser 3:41.678710308
5862024-09-27T23:14:01.137Zsys 13.640645436
5872024-09-27T23:14:01.137Ztrap 0.204203201
5882024-09-27T23:14:01.137Ztflt 0.010180589
5892024-09-27T23:14:01.137Zdflt 0.084131196
5902024-09-27T23:14:01.137Zkflt 0.001703680
5912024-09-27T23:14:01.137Zlock 8:33:30.951628856
5922024-09-27T23:14:01.137Zslp 33:45.475693328
5932024-09-27T23:14:01.138Zlat 11.693726450
5942024-09-27T23:14:01.138Zstop 0.129596959
5952024-09-27T23:14:01.138Z+ echo 'test-up-encrypted ends'
5962024-09-27T23:14:01.138Ztest-up-encrypted ends
5972024-09-27T23:14:06.140Zprocess exited: duration 582759 ms, exit code 0
5982024-09-27T23:14:06.140Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
5992024-09-27T23:15:06.180Zexec warning: : stdout descriptor may be held open by a background process; giving up!
6002024-09-27T23:15:06.180Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
6012024-09-27T23:15:06.191Zfound 14 output files
6022024-09-27T23:15:06.191Zuploading: /tmp/test_up-build/dsc-out.txt (35956 bytes)
6032024-09-27T23:15:07.203Zuploaded: /tmp/test_up-build/dsc-out.txt
6042024-09-27T23:15:07.212Zuploading: /tmp/test_up-build/test_up_out.txt (10491903 bytes)
6052024-09-27T23:15:08.364Zuploaded: /tmp/test_up-build/test_up_out.txt
6062024-09-27T23:15:08.364Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (253713 bytes)
6072024-09-27T23:15:09.386Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
6082024-09-27T23:15:09.386Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (144641 bytes)
6092024-09-27T23:15:10.403Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
6102024-09-27T23:15:10.406Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (251117 bytes)
6112024-09-27T23:15:11.425Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
6122024-09-27T23:15:11.427Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (139422 bytes)
6132024-09-27T23:15:12.442Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
6142024-09-27T23:15:12.442Zuploading: /tmp/debug/df.txt (1270 bytes)
6152024-09-27T23:15:13.451Zuploaded: /tmp/debug/df.txt
6162024-09-27T23:15:13.451Zuploading: /tmp/debug/dtrace.txt (495494 bytes)
6172024-09-27T23:15:14.481Zuploaded: /tmp/debug/dtrace.txt
6182024-09-27T23:15:14.484Zuploading: /tmp/debug/iostat.txt (193344 bytes)
6192024-09-27T23:15:14.499Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 193344 -> 195760
6202024-09-27T23:15:15.505Zuploaded: /tmp/debug/iostat.txt
6212024-09-27T23:15:15.507Zuploading: /tmp/debug/mpstat.txt (467843 bytes)
6222024-09-27T23:15:15.517Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 467843 -> 474422
6232024-09-27T23:15:16.524Zuploaded: /tmp/debug/mpstat.txt
6242024-09-27T23:15:16.527Zuploading: /tmp/debug/paging.txt (84992 bytes)
6252024-09-27T23:15:16.534Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 84992 -> 86320
6262024-09-27T23:15:17.541Zuploaded: /tmp/debug/paging.txt
6272024-09-27T23:15:17.544Zuploading: /tmp/debug/prstat.txt (903053 bytes)
6282024-09-27T23:15:17.577Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 903053 -> 917283
6292024-09-27T23:15:18.583Zuploaded: /tmp/debug/prstat.txt
6302024-09-27T23:15:18.586Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
6312024-09-27T23:15:19.591Zuploaded: /tmp/debug/psrinfo.txt
6322024-09-27T23:15:19.591Zuploading: /tmp/debug/upstairs-info.txt (95748 bytes)
6332024-09-27T23:15:20.605Zuploaded: /tmp/debug/upstairs-info.txt