01KV98D723CBM6245QETQJ4XDZ: test-up-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01KV98DR5HBZWJNPMQ0X7Z3CHK

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12026-06-16T22:34:23.358Zjob dependencies complete; ready to run (waiting for 12 m 28 s)
22026-06-16T22:36:59.262Zjob assigned to worker 01KV9982MJDX6QQ9HEMC6R6J26 [factory aws, i-022c7ae23fdd0b80a] (queued for 2 m 35 s)
32026-06-16T22:37:05.661Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42026-06-16T22:37:07.204Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52026-06-16T22:37:07.207Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62026-06-16T22:37:08.666Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72026-06-16T22:37:08.669Zdownloading input: /input/build/work/bins/crutest.gz
82026-06-16T22:37:10.399Zdownloaded input: /input/build/work/bins/crutest.gz
92026-06-16T22:37:10.399Zdownloading input: /input/build/work/bins/dsc.gz
102026-06-16T22:37:11.309Zdownloaded input: /input/build/work/bins/dsc.gz
112026-06-16T22:37:11.321Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122026-06-16T22:37:11.342Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132026-06-16T22:37:11.350Zdownloading input: /input/build/work/scripts/test_ds.sh
142026-06-16T22:37:11.368Zdownloaded input: /input/build/work/scripts/test_ds.sh
152026-06-16T22:37:11.372Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162026-06-16T22:37:11.399Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172026-06-16T22:37:11.399Zdownloading input: /input/build/work/scripts/test_repair.sh
182026-06-16T22:37:11.431Zdownloaded input: /input/build/work/scripts/test_repair.sh
192026-06-16T22:37:11.431Zdownloading input: /input/build/work/scripts/test_replay.sh
202026-06-16T22:37:11.460Zdownloaded input: /input/build/work/scripts/test_replay.sh
212026-06-16T22:37:11.461Zdownloading input: /input/build/work/scripts/test_up.sh
222026-06-16T22:37:11.489Zdownloaded input: /input/build/work/scripts/test_up.sh
232026-06-16T22:37:11.493Zdownloading input: /input/build/work/scripts/upstairs_info.d
242026-06-16T22:37:11.519Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252026-06-16T22:37:11.523Zdownloading input: /input/build/tmp/cargo-test-out.log
262026-06-16T22:37:12.052Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272026-06-16T22:37:12.052Zstarting task 0: "setup"
282026-06-16T22:37:12.057Z++ uname -s
292026-06-16T22:37:12.061Z+ kern=SunOS
302026-06-16T22:37:12.061Z+ build_user=build
312026-06-16T22:37:12.061Z+ build_uid=12345
322026-06-16T22:37:12.061Z+ work_dir=/work
332026-06-16T22:37:12.061Z+ input_dir=/input
342026-06-16T22:37:12.061Z+ [[ 0 == 12345 ]]
352026-06-16T22:37:12.061Z+ case "$kern" in
362026-06-16T22:37:12.061Z+ groupadd -g 12345 build
372026-06-16T22:37:12.064Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382026-06-16T22:37:14.067Z+ zfs create -o mountpoint=/work rpool/work
392026-06-16T22:37:14.428Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402026-06-16T22:37:14.432Z+ home_fs=zfs
412026-06-16T22:37:14.432Z+ [[ zfs == autofs ]]
422026-06-16T22:37:14.432Z+ mkdir -p /home/build
432026-06-16T22:37:14.432Z+ chown build:build /home/build /work
442026-06-16T22:37:16.432Z+ chmod 0700 /home/build /work
452026-06-16T22:37:16.435Zprocess exited: duration 4381 ms, exit code 0
 
462026-06-16T22:37:16.447Zstarting task 1: "authentication"
472026-06-16T22:37:16.465Zprocess exited: duration 20 ms, exit code 0
 
482026-06-16T22:37:16.471Zstarting task 2: "build"
492026-06-16T22:37:16.474Z+ banner cores
502026-06-16T22:37:16.477Z
512026-06-16T22:37:16.477Z #### #### ##### ###### ####
522026-06-16T22:37:16.477Z # # # # # # # #
532026-06-16T22:37:16.477Z # # # # # ##### ####
542026-06-16T22:37:16.477Z # # # ##### # #
552026-06-16T22:37:16.477Z # # # # # # # # #
562026-06-16T22:37:16.477Z #### #### # # ###### ####
572026-06-16T22:37:16.477Z
582026-06-16T22:37:16.477Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
592026-06-16T22:37:16.482Z+ echo 'input bins dir contains:'
602026-06-16T22:37:16.482Zinput bins dir contains:
612026-06-16T22:37:16.485Z+ ls -ltr /input/build/work/bins
622026-06-16T22:37:16.485Ztotal 1014844
632026-06-16T22:37:16.485Z-rw-r--r-- 1 root root 152775356 Jun 16 22:37 crucible-downstairs.gz
642026-06-16T22:37:16.485Z-rw-r--r-- 1 root root 145706593 Jun 16 22:37 crucible-hammer.gz
652026-06-16T22:37:16.485Z-rw-r--r-- 1 root root 151811257 Jun 16 22:37 crutest.gz
662026-06-16T22:37:16.485Z-rw-r--r-- 1 root root 68964813 Jun 16 22:37 dsc.gz
672026-06-16T22:37:16.485Z+ banner unpack
682026-06-16T22:37:16.485Z
692026-06-16T22:37:16.485Z # # # # ##### ## #### # #
702026-06-16T22:37:16.485Z # # ## # # # # # # # # #
712026-06-16T22:37:16.485Z # # # # # # # # # # ####
722026-06-16T22:37:16.485Z # # # # # ##### ###### # # #
732026-06-16T22:37:16.485Z # # # ## # # # # # # #
742026-06-16T22:37:16.485Z #### # # # # # #### # #
752026-06-16T22:37:16.485Z
762026-06-16T22:37:16.485Z+ mkdir -p /var/tmp/bins
772026-06-16T22:37:16.488Z+ for t in "$input/bins/"*.gz
782026-06-16T22:37:16.488Z++ basename /input/build/work/bins/crucible-downstairs.gz
792026-06-16T22:37:16.488Z+ b=crucible-downstairs.gz
802026-06-16T22:37:16.488Z+ b=crucible-downstairs
812026-06-16T22:37:16.488Z+ gunzip
822026-06-16T22:37:19.062Z+ chmod +x /var/tmp/bins/crucible-downstairs
832026-06-16T22:37:19.065Z+ for t in "$input/bins/"*.gz
842026-06-16T22:37:19.065Z++ basename /input/build/work/bins/crucible-hammer.gz
852026-06-16T22:37:19.065Z+ b=crucible-hammer.gz
862026-06-16T22:37:19.065Z+ b=crucible-hammer
872026-06-16T22:37:19.068Z+ gunzip
882026-06-16T22:37:21.463Z+ chmod +x /var/tmp/bins/crucible-hammer
892026-06-16T22:37:21.466Z+ for t in "$input/bins/"*.gz
902026-06-16T22:37:21.466Z++ basename /input/build/work/bins/crutest.gz
912026-06-16T22:37:21.469Z+ b=crutest.gz
922026-06-16T22:37:21.469Z+ b=crutest
932026-06-16T22:37:21.469Z+ gunzip
942026-06-16T22:37:23.999Z+ chmod +x /var/tmp/bins/crutest
952026-06-16T22:37:24.006Z+ for t in "$input/bins/"*.gz
962026-06-16T22:37:24.006Z++ basename /input/build/work/bins/dsc.gz
972026-06-16T22:37:24.006Z+ b=dsc.gz
982026-06-16T22:37:24.006Z+ b=dsc
992026-06-16T22:37:24.007Z+ gunzip
1002026-06-16T22:37:25.158Z+ chmod +x /var/tmp/bins/dsc
1012026-06-16T22:37:25.218Z+ export BINDIR=/var/tmp/bins
1022026-06-16T22:37:25.218Z+ BINDIR=/var/tmp/bins
1032026-06-16T22:37:25.219Z+ export RUST_BACKTRACE=1
1042026-06-16T22:37:25.219Z+ RUST_BACKTRACE=1
1052026-06-16T22:37:25.219Z+ jobpid=1088
1062026-06-16T22:37:25.219Z+ echo 'Setup debug logging'
1072026-06-16T22:37:25.219ZSetup debug logging
1082026-06-16T22:37:25.219Z+ mkdir /tmp/debug
1092026-06-16T22:37:25.219Z+ sleep 3600
1102026-06-16T22:37:25.219Z+ psrinfo -v
1112026-06-16T22:37:25.220Z+ df -h
1122026-06-16T22:37:25.220Z+ prstat -d d -mLc 1
1132026-06-16T22:37:25.220Z+ iostat -T d -xn 1
1142026-06-16T22:37:25.220Z+ mpstat -T d 1
1152026-06-16T22:37:25.220Z+ vmstat -T d -p 1
1162026-06-16T22:37:25.220Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1172026-06-16T22:37:25.220Z+ banner test_up_encrypted
1182026-06-16T22:37:25.220Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1192026-06-16T22:37:25.221Z
1202026-06-16T22:37:25.221Z ##### ###### #### ##### # # ##### ###### # #
1212026-06-16T22:37:25.221Z # # # # # # # # # ## #
1222026-06-16T22:37:25.221Z # ##### #### # # # # # ##### # # #
1232026-06-16T22:37:25.221Z # # # # # # ##### # # # #
1242026-06-16T22:37:25.221Z # # # # # # # # # # ##
1252026-06-16T22:37:25.221Z # ###### #### # ####### #### # ####### ###### # #
1262026-06-16T22:37:25.221Z
1272026-06-16T22:37:25.221Z+ ptime -m bash /input/build/work/scripts/test_up.sh -N encrypted
1282026-06-16T22:37:25.222Z/input/build/work
1292026-06-16T22:37:25.222ZTurn off color for downstairs dump
1302026-06-16T22:37:25.222ZUpstairs using key: 2SWQp76e4E01N+4XcCT9Z/Utlqioc4tIvLht6gtfiOU=
1312026-06-16T22:37:25.222Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1322026-06-16T22:37:25.222ZCreating 3 downstairs regions
1332026-06-16T22:37:25.412ZStarting 3 downstairs
1342026-06-16T22:37:25.435Zdsc started at PID: 1134
1352026-06-16T22:37:30.442ZDisable automatic restart on all downstairs
1362026-06-16T22:37:30.455Z
1372026-06-16T22:37:30.455ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1382026-06-16T22:37:30.458ZRunning test: span
1392026-06-16T22:37:34.511ZCompleted test: span
1402026-06-16T22:37:39.518ZRunning test: big
1412026-06-16T22:37:43.680ZCompleted test: big
1422026-06-16T22:37:48.687ZRunning test: dep
1432026-06-16T22:38:19.817ZCompleted test: dep
1442026-06-16T22:38:24.825ZRunning test: balloon
1452026-06-16T22:38:49.832ZCompleted test: balloon
1462026-06-16T22:38:54.839ZRunning test: deactivate
1472026-06-16T22:38:59.192ZCompleted test: deactivate
1482026-06-16T22:39:04.197ZRunning hammer
1492026-06-16T22:39:11.476ZRun repair tests
1502026-06-16T22:39:11.479Z/var/tmp/bins/crutest fill -g 31 -q --key 2SWQp76e4E01N+4XcCT9Z/Utlqioc4tIvLht6gtfiOU= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1512026-06-16T22:39:11.497Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.504136534Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"task":"crutest"}
1522026-06-16T22:39:11.501Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.504493157Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"task":"crutest"}
1532026-06-16T22:39:11.501Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.505091738Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"task":"crutest"}
1542026-06-16T22:39:11.501Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.505358746Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171}
1552026-06-16T22:39:11.501Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1f21c06f1a8568bef35f399f9603d7ed714f763\",\n git_commit_timestamp: \"2026-06-16T22:21:06.000000000Z\",\n git_branch: \"iliana/update-rustls\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.505395384Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171}
1562026-06-16T22:39:11.501Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.505404653Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171}
1572026-06-16T22:39:11.501Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.505417383Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171}
1582026-06-16T22:39:11.501Z{"msg":"Crucible 7efc0aa6-0606-488a-82dc-621680415b2c has session id: e9a47bb8-706c-4440-99d7-f2bc4b110119","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.505774386Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1592026-06-16T22:39:11.501Z{"msg":"Upstairs opts: Upstairs UUID: 7efc0aa6-0606-488a-82dc-621680415b2c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.505798495Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1602026-06-16T22:39:11.502Z{"msg":"Crucible stats registered with UUID: 7efc0aa6-0606-488a-82dc-621680415b2c","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.505809274Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1612026-06-16T22:39:11.502Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-06-16T22:39:10.505818804Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1622026-06-16T22:39:11.502Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.505951287Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"":"io task","client":"1","":"downstairs","session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1632026-06-16T22:39:11.502Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.505969897Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"":"io task","client":"0","":"downstairs","session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1642026-06-16T22:39:11.502Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.505980776Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"":"io task","client":"2","":"downstairs","session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1652026-06-16T22:39:11.502Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.506173507Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"task":"crutest"}
1662026-06-16T22:39:11.502Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.506186116Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171}
1672026-06-16T22:39:11.502Z{"msg":"7efc0aa6-0606-488a-82dc-621680415b2c active request set","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.506222924Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1682026-06-16T22:39:11.502Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.506235334Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"client":"0","":"downstairs","session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1692026-06-16T22:39:11.502Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.506249853Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"client":"1","":"downstairs","session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1702026-06-16T22:39:11.503Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.506259663Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"client":"2","":"downstairs","session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1712026-06-16T22:39:11.503Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.506270032Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"":"io task","client":"0","":"downstairs","session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1722026-06-16T22:39:11.503Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.506280782Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"":"io task","client":"2","":"downstairs","session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1732026-06-16T22:39:11.503Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.506291491Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"":"io task","client":"1","":"downstairs","session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1742026-06-16T22:39:11.503Z{"msg":"ds_connection connected from Ok(127.0.0.1:62871)","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.506355108Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"":"io task","client":"0","":"downstairs","session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1752026-06-16T22:39:11.503Z{"msg":"ds_connection connected from Ok(127.0.0.1:57543)","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.506385867Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"":"io task","client":"2","":"downstairs","session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1762026-06-16T22:39:11.503Z{"msg":"ds_connection connected from Ok(127.0.0.1:40815)","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.506408756Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"":"io task","client":"1","":"downstairs","session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1772026-06-16T22:39:11.503Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.507070054Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"client":"2","":"downstairs","session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1782026-06-16T22:39:11.503Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.507103153Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"client":"1","":"downstairs","session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1792026-06-16T22:39:11.503Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.507133611Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"client":"0","":"downstairs","session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1802026-06-16T22:39:11.503Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.507239026Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1812026-06-16T22:39:11.503Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.507312563Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1822026-06-16T22:39:11.506Z{"msg":"[0]R flush_numbers: [1497, 1497, 1497, 1497, 1497]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.507352751Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"":"downstairs","":"downstairs","session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1832026-06-16T22:39:11.506Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.50736413Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"":"downstairs","":"downstairs","session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1842026-06-16T22:39:11.506Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.50737504Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"":"downstairs","":"downstairs","session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1852026-06-16T22:39:11.506Z{"msg":"[1]R flush_numbers: [1497, 1497, 1497, 1497, 1497]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.507385949Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"":"downstairs","":"downstairs","session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1862026-06-16T22:39:11.506Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.507396489Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"":"downstairs","":"downstairs","session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1872026-06-16T22:39:11.506Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.507406878Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"":"downstairs","":"downstairs","session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1882026-06-16T22:39:11.506Z{"msg":"[2]R flush_numbers: [1497, 1497, 1497, 1497, 1497]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.507417558Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"":"downstairs","":"downstairs","session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1892026-06-16T22:39:11.506Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.507427897Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"":"downstairs","":"downstairs","session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1902026-06-16T22:39:11.507Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.507438217Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"":"downstairs","":"downstairs","session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1912026-06-16T22:39:11.507Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.507448426Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"":"downstairs","":"downstairs","session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1922026-06-16T22:39:11.507Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.507459216Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"":"downstairs","":"downstairs","session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1932026-06-16T22:39:11.507Z{"msg":"Next flush: 1498","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.507469285Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"":"downstairs","":"downstairs","session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1942026-06-16T22:39:11.507Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.507479655Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"":"downstairs","":"downstairs","session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1952026-06-16T22:39:11.507Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.507490574Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1962026-06-16T22:39:11.507Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.507499954Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"client":"0","":"downstairs","session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1972026-06-16T22:39:11.507Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.507510143Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"client":"1","":"downstairs","session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1982026-06-16T22:39:11.507Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.507520233Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"client":"2","":"downstairs","session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
1992026-06-16T22:39:11.507Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.507530822Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
2002026-06-16T22:39:11.507Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.507540652Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
2012026-06-16T22:39:11.507Z{"msg":"7efc0aa6-0606-488a-82dc-621680415b2c is now active with session: e9a47bb8-706c-4440-99d7-f2bc4b110119","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.507551651Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
2022026-06-16T22:39:11.507Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.507560981Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"session_id":"e9a47bb8-706c-4440-99d7-f2bc4b110119"}
2032026-06-16T22:39:11.507Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.50757103Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171}
2042026-06-16T22:39:11.507Z{"msg":"Activated sub_volume 7efc0aa6-0606-488a-82dc-621680415b2c","v":0,"name":"crucible","level":30,"Disk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2052026-06-16T22:39:11.507Ztime":"2026-06-16T22:39:10.50757894Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171}
2062026-06-16T22:39:11.507Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.507593549Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"task":"crutest"}
2072026-06-16T22:39:11.507Z{"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.507617468Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"task":"crutest"}
2082026-06-16T22:39:11.524ZRead and Verify all blocks (0..50 range:false)
2092026-06-16T22:39:11.548Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30----------------------------------------------------------------
2102026-06-16T22:39:11.548Z,"time":"2026-06-16T22:39:10.554969654Z","hostname":" Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2112026-06-16T22:39:11.548Zw-01KV9982MJDX6QQ9HEMC6R6J26"States:,"pid": Active1171 Active Active
2122026-06-16T22:39:11.548Z,"task":JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2132026-06-16T22:39:11.548Z"crutest"}
2142026-06-16T22:39:11.551Z{"msg":" 1002 Acked Read 50CLIENT: Tests done. All submitted work has been ACK'd","v" Done: Done0 Done, false"
2152026-06-16T22:39:11.551Zname":"crucible","level":30 STATES DS:0 DS:1 DS:2 TOTAL
2162026-06-16T22:39:11.551Z Sent 0 0 0 0
2172026-06-16T22:39:11.551Z," Done 1 time" 1 : 1 " 3
2182026-06-16T22:39:11.551Z2026-06-16T22:39:10.555016202Z" Skipped , 0 " 0 0 hostname" 0
2192026-06-16T22:39:11.551Z:" Error 0 0 0 w-01KV9982MJDX6QQ9HEMC6R6J26 0
2202026-06-16T22:39:11.552Z","pidLast Flush: ":1171Some(JobId(1001)) ,Some(JobId(1001)) "Some(JobId(1001)) task
2212026-06-16T22:39:11.552Z":"Downstairs last five completed:crutest"}
2222026-06-16T22:39:11.552Z 1001 1000
2232026-06-16T22:39:11.552ZUpstairs last five completed: 1002 1001 1000
2242026-06-16T22:39:11.552Z{"msg":"CLIENT: Up:0 ds:1 act:3","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:10.55505332Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"task":"crutest"}
2252026-06-16T22:39:15.550Z----------------------------------------------------------------
2262026-06-16T22:39:15.550Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2272026-06-16T22:39:15.550ZDownstairs last five completed: 1003 1002 1001 1000
2282026-06-16T22:39:15.550ZUpstairs last five completed: 1003 1002 1001 1000
2292026-06-16T22:39:15.553Z{"msg":"CLIENT: Up:0 ds:0 act:3","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.557058775Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"task":"crutest"}
2302026-06-16T22:39:15.553Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.557115072Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171,"task":"crutest"}
2312026-06-16T22:39:15.553Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-06-16T22:39:14.557135901Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1171}
2322026-06-16T22:39:15.558ZRepair setup passed
2332026-06-16T22:39:15.561ZCopy the region for /var/tmp/test_up-build/8830
2342026-06-16T22:39:15.563Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2352026-06-16T22:39:15.564Z/var/tmp/bins/crutest fill -g 32 -q --key 2SWQp76e4E01N+4XcCT9Z/Utlqioc4tIvLht6gtfiOU= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2362026-06-16T22:39:15.582Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.588692559Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"task":"crutest"}
2372026-06-16T22:39:15.585Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.589057682Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"task":"crutest"}
2382026-06-16T22:39:15.585Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.589777247Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"task":"crutest"}
2392026-06-16T22:39:15.585Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.590021586Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177}
2402026-06-16T22:39:15.585Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1f21c06f1a8568bef35f399f9603d7ed714f763\",\n git_commit_timestamp: \"2026-06-16T22:21:06.000000000Z\",\n git_branch: \"iliana/update-rustls\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.590049734Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177}
2412026-06-16T22:39:15.585Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.590062614Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177}
2422026-06-16T22:39:15.585Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.590074783Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177}
2432026-06-16T22:39:15.585Z{"msg":"Crucible 936510a1-553c-43d6-b037-bde5b1d1d494 has session id: 5997f22d-65f0-4ea5-8ec5-effb81b249d0","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.590539901Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2442026-06-16T22:39:15.585Z{"msg":"Upstairs opts: Upstairs UUID: 936510a1-553c-43d6-b037-bde5b1d1d494, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.59056348Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2452026-06-16T22:39:15.585Z{"msg":"Crucible stats registered with UUID: 936510a1-553c-43d6-b037-bde5b1d1d494","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.590574689Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2462026-06-16T22:39:15.585Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-06-16T22:39:14.590584429Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2472026-06-16T22:39:15.585Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.590722632Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"":"io task","client":"1","":"downstairs","session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2482026-06-16T22:39:15.585Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.590741622Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"":"io task","client":"0","":"downstairs","session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2492026-06-16T22:39:15.585Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.590752251Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"":"io task","client":"2","":"downstairs","session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2502026-06-16T22:39:15.585Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.590934682Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"task":"crutest"}
2512026-06-16T22:39:15.585Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.590951592Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177}
2522026-06-16T22:39:15.585Z{"msg":"936510a1-553c-43d6-b037-bde5b1d1d494 active request set","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.591006059Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2532026-06-16T22:39:15.585Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.591016388Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"client":"0","":"downstairs","session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2542026-06-16T22:39:15.586Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.591026698Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"client":"1","":"downstairs","session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2552026-06-16T22:39:15.587Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.591044377Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"client":"2","":"downstairs","session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2562026-06-16T22:39:15.587Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.591054077Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"":"io task","client":"0","":"downstairs","session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2572026-06-16T22:39:15.587Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.591063936Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"":"io task","client":"2","":"downstairs","session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2582026-06-16T22:39:15.587Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.591074826Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"":"io task","client":"1","":"downstairs","session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2592026-06-16T22:39:15.587Z{"msg":"ds_connection connected from Ok(127.0.0.1:46172)","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.591128853Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"":"io task","client":"2","":"downstairs","session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2602026-06-16T22:39:15.587Z{"msg":"ds_connection connected from Ok(127.0.0.1:35411)","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.591140233Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"":"io task","client":"0","":"downstairs","session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2612026-06-16T22:39:15.587Z{"msg":"ds_connection connected from Ok(127.0.0.1:56093)","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.591150542Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"":"io task","client":"1","":"downstairs","session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2622026-06-16T22:39:15.590Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.591964343Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"client":"2","":"downstairs","session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2632026-06-16T22:39:15.590Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.592010831Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"client":"0","":"downstairs","session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2642026-06-16T22:39:15.590Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.59202746Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"client":"1","":"downstairs","session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2652026-06-16T22:39:15.590Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.592208022Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2662026-06-16T22:39:15.590Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.592222111Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2672026-06-16T22:39:15.590Z{"msg":"[0]R flush_numbers: [1498, 1498, 1498, 1498, 1498]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.59225341Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"":"downstairs","":"downstairs","session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2682026-06-16T22:39:15.590Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.592268279Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"":"downstairs","":"downstairs","session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2692026-06-16T22:39:15.590Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.592278168Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"":"downstairs","":"downstairs","session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2702026-06-16T22:39:15.590Z{"msg":"[1]R flush_numbers: [1498, 1498, 1498, 1498, 1498]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.592292388Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"":"downstairs","":"downstairs","session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2712026-06-16T22:39:15.590Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.592302027Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"":"downstairs","":"downstairs","session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2722026-06-16T22:39:15.590Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.592316387Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"":"downstairs","":"downstairs","session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2732026-06-16T22:39:15.590Z{"msg":"[2]R flush_numbers: [1498, 1498, 1498, 1498, 1498]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.592326756Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"":"downstairs","":"downstairs","session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2742026-06-16T22:39:15.590Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.592340565Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"":"downstairs","":"downstairs","session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2752026-06-16T22:39:15.591Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.592350455Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"":"downstairs","":"downstairs","session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2762026-06-16T22:39:15.591Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.592364104Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"":"downstairs","":"downstairs","session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2772026-06-16T22:39:15.591Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.592373754Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"":"downstairs","":"downstairs","session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2782026-06-16T22:39:15.591Z{"msg":"Next flush: 1499","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.592383673Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"":"downstairs","":"downstairs","session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2792026-06-16T22:39:15.591Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.592397103Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"":"downstairs","":"downstairs","session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2802026-06-16T22:39:15.591Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.592406812Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2812026-06-16T22:39:15.591Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.592415982Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"client":"0","":"downstairs","session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2822026-06-16T22:39:15.591Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.592429841Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"client":"1","":"downstairs","session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2832026-06-16T22:39:15.591Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.592439351Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"client":"2","":"downstairs","session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2842026-06-16T22:39:15.591Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.59245315Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2852026-06-16T22:39:15.591Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.59246254Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2862026-06-16T22:39:15.591Z{"msg":"936510a1-553c-43d6-b037-bde5b1d1d494 is now active with session: 5997f22d-65f0-4ea5-8ec5-effb81b249d0","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.592476739Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2872026-06-16T22:39:15.591Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.592518137Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"session_id":"5997f22d-65f0-4ea5-8ec5-effb81b249d0"}
2882026-06-16T22:39:15.591Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.592530516Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177}
2892026-06-16T22:39:15.591Z{"msg":"Activated sub_volume 936510a1-553c-43d6-b037-bde5b1d1d494","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.592538776Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177}
2902026-06-16T22:39:15.591Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.592547506Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"task":"crutest"}
2912026-06-16T22:39:15.591ZDisk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2922026-06-16T22:39:15.591ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
2932026-06-16T22:39:15.592Z{"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.592885099Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"task":"crutest"}
2942026-06-16T22:39:15.609ZRead and Verify all blocks (0..50 range:false)
2952026-06-16T22:39:15.633Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.6399258Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"task":"crutest----------------------------------------------------------------
2962026-06-16T22:39:15.633Z"}
2972026-06-16T22:39:15.633Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
2982026-06-16T22:39:15.633Z{States:"msg":" Active Active ActiveCLIENT: Tests done. All submitted work has been ACK'd
2992026-06-16T22:39:15.633Z",JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
3002026-06-16T22:39:15.633Z"v":0,"name":"crucible","level":30 1002 Acked Read 50," Donetime Done" Done:" false
3012026-06-16T22:39:15.633Z2026-06-16T22:39:14.639956629Z","hostname":" STATES DS:0 DS:1 DS:2 TOTAL
3022026-06-16T22:39:15.633Zw-01KV9982MJDX6QQ9HEMC6R6J26", Sent "pid" 0 : 0 1177 0 ," 0
3032026-06-16T22:39:15.633Ztask": Done " 1 crutest 1 " 1 } 3
3042026-06-16T22:39:15.633Z
3052026-06-16T22:39:15.633Z Skipped 0 0 0 0
3062026-06-16T22:39:15.633Z Error 0 0 0 0
3072026-06-16T22:39:15.636ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
3082026-06-16T22:39:15.636ZDownstairs last five completed: 1001 1000
3092026-06-16T22:39:15.636ZUpstairs last five completed: 1002 1001 1000
3102026-06-16T22:39:15.636Z{"msg":"CLIENT: Up:0 ds:1 act:3","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:14.640010516Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"task":"crutest"}
3112026-06-16T22:39:19.634Z----------------------------------------------------------------
3122026-06-16T22:39:19.634Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
3132026-06-16T22:39:19.634ZDownstairs last five completed: 1003 1002 1001 1000
3142026-06-16T22:39:19.635ZUpstairs last five completed: 1003 1002 1001 1000
3152026-06-16T22:39:19.637Z{"msg":"CLIENT: Up:0 ds:0 act:3","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:18.641130073Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"task":"crutest"}
3162026-06-16T22:39:19.637Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:18.6411997Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177,"task":"crutest"}
3172026-06-16T22:39:19.638Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-06-16T22:39:18.641220019Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1177}
3182026-06-16T22:39:19.640ZRepair part 1 passed
3192026-06-16T22:39:19.640Z
3202026-06-16T22:39:19.640ZKill the current downstairs
3212026-06-16T22:39:19.683ZDownstairs 2 stopped
3222026-06-16T22:39:19.683Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
3232026-06-16T22:39:19.686ZNow put back the original so we have a mismatch
3242026-06-16T22:39:19.686Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
3252026-06-16T22:39:19.686ZRestart downstairs with old directory
3262026-06-16T22:39:19.699Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
3272026-06-16T22:39:19.711ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3282026-06-16T22:39:19.714Z 0 000-009 32 32 31 1499 1499 1498 F F F <---
3292026-06-16T22:39:19.714Z 1 010-019 32 32 31 1499 1499 1498 F F F <---
3302026-06-16T22:39:19.714Z 2 020-029 32 32 31 1499 1499 1498 F F F <---
3312026-06-16T22:39:19.714Z 3 030-039 32 32 31 1499 1499 1498 F F F <---
3322026-06-16T22:39:19.714Z 4 040-049 32 32 31 1499 1499 1498 F F F <---
3332026-06-16T22:39:19.714ZMax gen: 32, Max flush: 1499
3342026-06-16T22:39:19.920ZError: Difference in extent metadata found!
3352026-06-16T22:39:19.920Z
3362026-06-16T22:39:19.920ZStack backtrace:
3372026-06-16T22:39:19.920Z 0: anyhow::error::<impl anyhow::Error>::msg
3382026-06-16T22:39:19.920Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14
3392026-06-16T22:39:19.920Z 1: anyhow::__private::format_err
3402026-06-16T22:39:19.920Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13
3412026-06-16T22:39:19.939Zdump test found error as expected
3422026-06-16T22:39:19.939Z
3432026-06-16T22:39:19.939Z
3442026-06-16T22:39:19.941Z/var/tmp/bins/crutest verify --range -g 33 -q --key 2SWQp76e4E01N+4XcCT9Z/Utlqioc4tIvLht6gtfiOU= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
3452026-06-16T22:39:19.959Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:18.965197177Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"task":"crutest"}
3462026-06-16T22:39:19.963Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:18.965750111Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"task":"crutest"}
3472026-06-16T22:39:19.963Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:18.966625059Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"task":"crutest"}
3482026-06-16T22:39:19.963Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:18.966904466Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185}
3492026-06-16T22:39:19.963Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1f21c06f1a8568bef35f399f9603d7ed714f763\",\n git_commit_timestamp: \"2026-06-16T22:21:06.000000000Z\",\n git_branch: \"iliana/update-rustls\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:18.966938384Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185}
3502026-06-16T22:39:19.963Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:18.966951484Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185}
3512026-06-16T22:39:19.963Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:18.966964033Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185}
3522026-06-16T22:39:19.965Z{"msg":"Crucible 288a9eea-6f28-4cea-acd8-9c142d275f18 has session id: dcbd61e6-87b9-472d-903e-f2c36410b3c9","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:18.96745162Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3532026-06-16T22:39:19.965Z{"msg":"Upstairs opts: Upstairs UUID: 288a9eea-6f28-4cea-acd8-9c142d275f18, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:18.967489488Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3542026-06-16T22:39:19.965Z{"msg":"Crucible stats registered with UUID: 288a9eea-6f28-4cea-acd8-9c142d275f18","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:18.967501187Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3552026-06-16T22:39:19.965Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-06-16T22:39:18.967515217Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3562026-06-16T22:39:19.965Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:18.967632981Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3572026-06-16T22:39:19.965Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:18.967675399Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"io task","client":"1","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3582026-06-16T22:39:19.965Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:18.967687429Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3592026-06-16T22:39:19.965Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:18.967975585Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"task":"crutest"}
3602026-06-16T22:39:19.965Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:18.967996284Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185}
3612026-06-16T22:39:19.965Z{"msg":"288a9eea-6f28-4cea-acd8-9c142d275f18 active request set","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:18.968036562Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3622026-06-16T22:39:19.965Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:18.968050801Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"client":"0","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3632026-06-16T22:39:19.965Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:18.968060761Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"client":"1","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3642026-06-16T22:39:19.966Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:18.96807467Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"client":"2","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3652026-06-16T22:39:19.966Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:18.96808461Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3662026-06-16T22:39:19.966Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:18.968095289Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3672026-06-16T22:39:19.966Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:18.968105819Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"io task","client":"1","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3682026-06-16T22:39:19.966Z{"msg":"ds_connection connected from Ok(127.0.0.1:42173)","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:18.968177195Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"io task","client":"1","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3692026-06-16T22:39:19.966Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-06-16T22:39:18.968200864Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3702026-06-16T22:39:19.966Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-06-16T22:39:18.968217253Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3712026-06-16T22:39:19.966Z{"msg":"ds_connection connected from Ok(127.0.0.1:33925)","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:18.968232003Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3722026-06-16T22:39:19.966Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-06-16T22:39:18.968267401Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3732026-06-16T22:39:19.966Z{"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:18.968403464Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3742026-06-16T22:39:19.966Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:18.968435313Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3752026-06-16T22:39:19.969Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:18.968945499Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"client":"1","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3762026-06-16T22:39:19.969Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:18.968986247Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"client":"0","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3772026-06-16T22:39:19.969Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:18.96913029Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3782026-06-16T22:39:19.969Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:18.969144449Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3792026-06-16T22:39:29.964Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.970343082Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3802026-06-16T22:39:29.969Z{"msg":"ds_connection connected from Ok(127.0.0.1:46076)","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.970517584Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3812026-06-16T22:39:29.969Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.971883889Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"client":"2","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3822026-06-16T22:39:29.969Z{"msg":"[0]R flush_numbers: [1499, 1499, 1499, 1499, 1499]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.972211433Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"downstairs","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3832026-06-16T22:39:29.969Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.972242152Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"downstairs","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3842026-06-16T22:39:29.969Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.972261041Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"downstairs","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3852026-06-16T22:39:29.969Z{"msg":"[1]R flush_numbers: [1499, 1499, 1499, 1499, 1499]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.97228772Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"downstairs","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3862026-06-16T22:39:29.969Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.972306409Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"downstairs","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3872026-06-16T22:39:29.969Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.972332377Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"downstairs","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3882026-06-16T22:39:29.969Z{"msg":"[2]R flush_numbers: [1498, 1498, 1498, 1498, 1498]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.972358236Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"downstairs","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3892026-06-16T22:39:29.969Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.972377165Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"downstairs","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3902026-06-16T22:39:29.969Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.972402984Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"downstairs","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3912026-06-16T22:39:29.969Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.972420213Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"downstairs","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3922026-06-16T22:39:29.969Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.972447332Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"downstairs","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3932026-06-16T22:39:29.969Z{"msg":"Next flush: 1500","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.972471441Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"downstairs","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3942026-06-16T22:39:29.969Z{"msg":"extent 0 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1499, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1499, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1498, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.97249773Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"mend","":"downstairs","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3952026-06-16T22:39:29.970Z{"msg":"extent:0 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1499, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1499, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1498, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.972532488Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"mend","":"downstairs","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3962026-06-16T22:39:29.970Z{"msg":"found dest for source 0 for extent at index 0 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.972560777Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"mend","":"downstairs","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3972026-06-16T22:39:29.970Z{"msg":"extent 1 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1499, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1499, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1498, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.972587025Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"mend","":"downstairs","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3982026-06-16T22:39:29.970Z{"msg":"extent:1 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1499, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1499, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1498, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.972619304Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"mend","":"downstairs","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
3992026-06-16T22:39:29.970Z{"msg":"found dest for source 0 for extent at index 1 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.972647243Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"mend","":"downstairs","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
4002026-06-16T22:39:29.970Z{"msg":"extent 2 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1499, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1499, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1498, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.972673511Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"mend","":"downstairs","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
4012026-06-16T22:39:29.970Z{"msg":"extent:2 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1499, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1499, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1498, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.97270692Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"mend","":"downstairs","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
4022026-06-16T22:39:29.971Z{"msg":"found dest for source 0 for extent at index 2 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.972734918Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"mend","":"downstairs","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
4032026-06-16T22:39:29.971Z{"msg":"extent 3 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1499, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1499, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1498, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.972767307Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"mend","":"downstairs","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
4042026-06-16T22:39:29.972Z{"msg":"extent:3 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1499, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1499, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1498, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.972799765Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"mend","":"downstairs","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
4052026-06-16T22:39:29.972Z{"msg":"found dest for source 0 for extent at index 3 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.972827724Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"mend","":"downstairs","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
4062026-06-16T22:39:29.972Z{"msg":"extent 4 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1499, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1499, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1498, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.972853333Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"mend","":"downstairs","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
4072026-06-16T22:39:29.972Z{"msg":"extent:4 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1499, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1499, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1498, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.972885721Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"mend","":"downstairs","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
4082026-06-16T22:39:29.972Z{"msg":"found dest for source 0 for extent at index 4 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.97291312Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"mend","":"downstairs","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
4092026-06-16T22:39:29.972Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.972941688Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"client":"0","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
4102026-06-16T22:39:29.972Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.972968207Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"client":"1","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
4112026-06-16T22:39:29.972Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.972987946Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"client":"2","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
4122026-06-16T22:39:29.972Z{"msg":"Full repair list: {ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.973026844Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"downstairs","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
4132026-06-16T22:39:29.972Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 1500, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 1500, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 1500, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 1500, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 1500, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.973195766Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"downstairs","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
4142026-06-16T22:39:29.972Z{"msg":"starting reconciliation d4d3f0f0-ecde-4004-ac0e-915056ee2cec: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.973216635Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"downstairs","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
4152026-06-16T22:39:29.972Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.973234715Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"client":"2","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
4162026-06-16T22:39:29.978Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.984241161Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"client":"2","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
4172026-06-16T22:39:29.981Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.98636556Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"client":"2","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
4182026-06-16T22:39:29.983Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.988397463Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"client":"2","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
4192026-06-16T22:39:29.986Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.990241105Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"client":"2","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
4202026-06-16T22:39:29.986Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.991950944Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"":"downstairs","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
4212026-06-16T22:39:29.986Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.991970983Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"client":"0","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
4222026-06-16T22:39:29.986Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.991982192Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"client":"1","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
4232026-06-16T22:39:29.986Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.991992952Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"client":"2","":"downstairs","session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
4242026-06-16T22:39:29.986Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.992007971Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
4252026-06-16T22:39:29.986Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.992018Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
4262026-06-16T22:39:29.986Z{"msg":"288a9eea-6f28-4cea-acd8-9c142d275f18 is now active with session: dcbd61e6-87b9-472d-903e-f2c36410b3c9","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.99203376Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"session_id":"dcbd61e6-87b9-472d-903e-f2c36410b3c9"}
4272026-06-16T22:39:29.986Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.992048019Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185}
4282026-06-16T22:39:29.986Z{"msg":"Activated sub_volume 288a9eea-6f28-4cea-acd8-9c142d275f18","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.992068428Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185}
4292026-06-16T22:39:29.986Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:28.992143275Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"task":"crutest"}
4302026-06-16T22:39:29.986ZDisk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
4312026-06-16T22:39:29.986ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4322026-06-16T22:39:29.989ZRead and Verify all blocks (0..50 range:true)
4332026-06-16T22:39:30.011Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:29.016807131Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"task":"crutest"}
4342026-06-16T22:39:30.011Z{"msg":"Verify test completed","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:29.016828569Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"task":"crutest"}
4352026-06-16T22:39:30.014Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:29.016839269Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"task":"crutest"}
4362026-06-16T22:39:30.014Z{"msg":"CLIENT: Tests done. All submitted work has been ACK'd","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:29.016848839Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"task":"crutest"}
4372026-06-16T22:39:30.014Z----------------------------------------------------------------
4382026-06-16T22:39:30.014Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
4392026-06-16T22:39:30.014ZStates: Active Active Active
4402026-06-16T22:39:30.014ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4412026-06-16T22:39:30.014Z 1000 Acked Read 50 Done Done Done false
4422026-06-16T22:39:30.014Z STATES DS:0 DS:1 DS:2 TOTAL
4432026-06-16T22:39:30.014Z Sent 0 0 0 0
4442026-06-16T22:39:30.014Z Done 1 1 1 3
4452026-06-16T22:39:30.014Z Skipped 0 0 0 0
4462026-06-16T22:39:30.014Z Error 0 0 0 0
4472026-06-16T22:39:30.014ZLast Flush: None None None
4482026-06-16T22:39:30.014ZDownstairs last five completed:
4492026-06-16T22:39:30.015ZUpstairs last five completed: 1000
4502026-06-16T22:39:30.015Z{"msg":"CLIENT: Up:0 ds:1 act:3","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:29.016954843Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"task":"crutest"}
4512026-06-16T22:39:34.012Z----------------------------------------------------------------
4522026-06-16T22:39:34.012Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
4532026-06-16T22:39:34.012ZDownstairs last five completed: 1001 1000
4542026-06-16T22:39:34.012ZUpstairs last five completed: 1001 1000
4552026-06-16T22:39:34.015Z{"msg":"CLIENT: Up:0 ds:0 act:3","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:33.017548195Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"task":"crutest"}
4562026-06-16T22:39:34.015Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-06-16T22:39:33.017589063Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185,"task":"crutest"}
4572026-06-16T22:39:34.015Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-06-16T22:39:33.017610152Z","hostname":"w-01KV9982MJDX6QQ9HEMC6R6J26","pid":1185}
4582026-06-16T22:39:34.018ZRepair part 2 passed
4592026-06-16T22:39:34.018Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4602026-06-16T22:39:34.031ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4612026-06-16T22:39:34.031Z 0 000-009 32 32 32 1499 1499 1499 F F F
4622026-06-16T22:39:34.031Z 1 010-019 32 32 32 1499 1499 1499 F F F
4632026-06-16T22:39:34.033Z 2 020-029 32 32 32 1499 1499 1499 F F F
4642026-06-16T22:39:34.033Z 3 030-039 32 32 32 1499 1499 1499 F F F
4652026-06-16T22:39:34.033Z 4 040-049 32 32 32 1499 1499 1499 F F F
4662026-06-16T22:39:34.033ZMax gen: 32, Max flush: 1499
4672026-06-16T22:39:34.033Zdump test passed
4682026-06-16T22:39:34.034Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
4692026-06-16T22:39:34.044Z Extent 1
4702026-06-16T22:39:34.044ZGEN 32 32 32
4712026-06-16T22:39:34.044ZFLUSH_ID 1499 1499 1499
4722026-06-16T22:39:34.044ZDIRTY
4732026-06-16T22:39:34.044Z
4742026-06-16T22:39:34.044ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
4752026-06-16T22:39:34.046Z010 A A A A A A
4762026-06-16T22:39:34.047Z011 A A A A A A
4772026-06-16T22:39:34.047Z012 A A A A A A
4782026-06-16T22:39:34.047Z013 A A A A A A
4792026-06-16T22:39:34.050Z014 A A A A A A
4802026-06-16T22:39:34.050Z015 A A A A A A
4812026-06-16T22:39:34.050Z016 A A A A A A
4822026-06-16T22:39:34.050Z017 A A A A A A
4832026-06-16T22:39:34.050Z018 A A A A A A
4842026-06-16T22:39:34.050Z019 A A A A A A
4852026-06-16T22:39:34.053Zdump extent test passed
4862026-06-16T22:39:34.053Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
4872026-06-16T22:39:34.062ZExtent 2 Block in extent 0 Actual block 20
4882026-06-16T22:39:34.062Z
4892026-06-16T22:39:34.064Z DATA SHA256 VER
4902026-06-16T22:39:34.064Z------ ---------------------------------------------------------------- ---
4912026-06-16T22:39:34.064Z 0 7e4a0291a9b96183296d34cbe2379cc05e27ce4ffa2269e133a067752804e33a A
4922026-06-16T22:39:34.064Z 1 7e4a0291a9b96183296d34cbe2379cc05e27ce4ffa2269e133a067752804e33a A
4932026-06-16T22:39:34.064Z 2 7e4a0291a9b96183296d34cbe2379cc05e27ce4ffa2269e133a067752804e33a A
4942026-06-16T22:39:34.065Z
4952026-06-16T22:39:34.065ZNONCES 0 1 2 DIFF
4962026-06-16T22:39:34.065Z------ ------------------------ ------------------------ ------------------------ -----
4972026-06-16T22:39:34.065Z 0 517b40e13d26b9d0559b08de 517b40e13d26b9d0559b08de 517b40e13d26b9d0559b08de
4982026-06-16T22:39:34.065Z
4992026-06-16T22:39:34.065Z TAGS 0 1 2 DIFF
5002026-06-16T22:39:34.065Z------ -------------------------------- -------------------------------- -------------------------------- -----
5012026-06-16T22:39:34.065Z 0 ae64532932c40aecc51058701f2ef139 ae64532932c40aecc51058701f2ef139 ae64532932c40aecc51058701f2ef139
5022026-06-16T22:39:34.065Z
5032026-06-16T22:39:34.065ZHASHES 0 1 2 DIFF
5042026-06-16T22:39:34.065Z------ ---------------- ---------------- ---------------- -----
5052026-06-16T22:39:34.065Z
5062026-06-16T22:39:34.204Zdump block test passed
5072026-06-16T22:39:34.204ZInitial upstairs tests have completed, stopping all downstairs
5082026-06-16T22:39:39.086ZCreating 4 larger downstairs regions
5092026-06-16T22:39:39.206ZStarting 4 downstairs
5102026-06-16T22:39:44.213Zdsc restarted at PID: 1198
5112026-06-16T22:39:44.272ZNow do the replace-reconcile test
5122026-06-16T22:39:44.285ZUsing 8840 for the replacement port
5132026-06-16T22:43:28.612ZCompleted test: replace-reconcile
5142026-06-16T22:43:28.616ZNow do the replace-before-active test
5152026-06-16T22:46:45.401ZCompleted test: replace-before-active
5162026-06-16T22:46:45.402ZAll tests have completed, stopping all downstairs
5172026-06-16T22:46:45.418Z
5182026-06-16T22:46:45.418ZAll Tests have passed
5192026-06-16T22:46:45.421Z9:20 Test duration
5202026-06-16T22:46:45.421Z
5212026-06-16T22:46:45.421Zreal 9:20.242038579
5222026-06-16T22:46:45.421Zuser 6:45.733742464
5232026-06-16T22:46:45.421Zsys 5.554335076
5242026-06-16T22:46:45.421Ztrap 0.187088020
5252026-06-16T22:46:45.421Ztflt 0.026917834
5262026-06-16T22:46:45.421Zdflt 0.010981181
5272026-06-16T22:46:45.421Zkflt 0.001168907
5282026-06-16T22:46:45.421Zlock 5:50:14.785726996
5292026-06-16T22:46:45.422Zslp 26:56.001207419
5302026-06-16T22:46:45.422Zlat 28.026778027
5312026-06-16T22:46:45.422Zstop 0.010764484
5322026-06-16T22:46:45.422Ztest-up-encrypted ends
5332026-06-16T22:46:45.422Z+ echo 'test-up-encrypted ends'
5342026-06-16T22:46:50.424Zprocess exited: duration 568917 ms, exit code 0
5352026-06-16T22:46:50.424Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
5362026-06-16T22:47:50.428Zexec warning: : stdout descriptor may be held open by a background process; giving up!
5372026-06-16T22:47:50.428Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
5382026-06-16T22:47:50.440Zfound 14 output files
5392026-06-16T22:47:50.446Zuploading: /tmp/test_up-build/dsc-out.txt (48257 bytes)
5402026-06-16T22:47:51.457Zuploaded: /tmp/test_up-build/dsc-out.txt
5412026-06-16T22:47:51.457Zuploading: /tmp/test_up-build/test_up_out.txt (22001796 bytes)
5422026-06-16T22:47:52.611Zuploaded: /tmp/test_up-build/test_up_out.txt
5432026-06-16T22:47:52.611Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (2497 bytes)
5442026-06-16T22:47:53.619Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
5452026-06-16T22:47:53.622Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (548316 bytes)
5462026-06-16T22:47:54.638Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
5472026-06-16T22:47:54.640Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (978625 bytes)
5482026-06-16T22:47:55.657Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
5492026-06-16T22:47:55.657Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (543062 bytes)
5502026-06-16T22:47:56.675Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
5512026-06-16T22:47:56.675Zuploading: /tmp/debug/df.txt (1270 bytes)
5522026-06-16T22:47:57.683Zuploaded: /tmp/debug/df.txt
5532026-06-16T22:47:57.686Zuploading: /tmp/debug/dtrace.txt (616298 bytes)
5542026-06-16T22:47:58.701Zuploaded: /tmp/debug/dtrace.txt
5552026-06-16T22:47:58.704Zuploading: /tmp/debug/iostat.txt (186073 bytes)
5562026-06-16T22:47:58.710Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 186073 -> 188449
5572026-06-16T22:47:59.717Zuploaded: /tmp/debug/iostat.txt
5582026-06-16T22:47:59.717Zuploading: /tmp/debug/mpstat.txt (454650 bytes)
5592026-06-16T22:47:59.729Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 454650 -> 461184
5602026-06-16T22:48:00.738Zuploaded: /tmp/debug/mpstat.txt
5612026-06-16T22:48:00.738Zuploading: /tmp/debug/paging.txt (80066 bytes)
5622026-06-16T22:48:00.745Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 80066 -> 81344
5632026-06-16T22:48:01.752Zuploaded: /tmp/debug/paging.txt
5642026-06-16T22:48:01.755Zuploading: /tmp/debug/prstat.txt (864780 bytes)
5652026-06-16T22:48:01.766Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 864780 -> 878966
5662026-06-16T22:48:02.772Zuploaded: /tmp/debug/prstat.txt
5672026-06-16T22:48:02.774Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
5682026-06-16T22:48:03.781Zuploaded: /tmp/debug/psrinfo.txt
5692026-06-16T22:48:03.785Zuploading: /tmp/debug/upstairs-info.txt (67711 bytes)
5702026-06-16T22:48:04.793Zuploaded: /tmp/debug/upstairs-info.txt