01KY0EQKZ83A9V881XVXBCMEPD: test-up-unencrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01KY0ER3ZDS5DKFM57WS191RXW

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12026-07-20T19:18:20.260Zjob dependencies complete; ready to run (waiting for 11 m 58 s)
22026-07-20T19:19:19.733Zjob assigned to worker 01KY0FEA4FNRZ5VB7NM1FCWT0R [factory aws, i-07e62c917d72f9537] (queued for 59 s)
32026-07-20T19:19:26.605Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42026-07-20T19:19:28.775Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52026-07-20T19:19:28.783Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62026-07-20T19:19:30.331Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72026-07-20T19:19:30.334Zdownloading input: /input/build/work/bins/crutest.gz
82026-07-20T19:19:32.390Zdownloaded input: /input/build/work/bins/crutest.gz
92026-07-20T19:19:32.395Zdownloading input: /input/build/work/bins/dsc.gz
102026-07-20T19:19:33.344Zdownloaded input: /input/build/work/bins/dsc.gz
112026-07-20T19:19:33.348Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122026-07-20T19:19:33.376Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132026-07-20T19:19:33.381Zdownloading input: /input/build/work/scripts/test_ds.sh
142026-07-20T19:19:33.404Zdownloaded input: /input/build/work/scripts/test_ds.sh
152026-07-20T19:19:33.412Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162026-07-20T19:19:33.436Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172026-07-20T19:19:33.436Zdownloading input: /input/build/work/scripts/test_repair.sh
182026-07-20T19:19:33.462Zdownloaded input: /input/build/work/scripts/test_repair.sh
192026-07-20T19:19:33.462Zdownloading input: /input/build/work/scripts/test_replay.sh
202026-07-20T19:19:33.497Zdownloaded input: /input/build/work/scripts/test_replay.sh
212026-07-20T19:19:33.497Zdownloading input: /input/build/work/scripts/test_up.sh
222026-07-20T19:19:33.529Zdownloaded input: /input/build/work/scripts/test_up.sh
232026-07-20T19:19:33.529Zdownloading input: /input/build/work/scripts/upstairs_info.d
242026-07-20T19:19:33.609Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252026-07-20T19:19:33.613Zdownloading input: /input/build/tmp/cargo-test-out.log
262026-07-20T19:19:34.251Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272026-07-20T19:19:34.251Zstarting task 0: "setup"
282026-07-20T19:19:34.255Z++ uname -s
292026-07-20T19:19:34.260Z+ kern=SunOS
302026-07-20T19:19:34.260Z+ build_user=build
312026-07-20T19:19:34.260Z+ build_uid=12345
322026-07-20T19:19:34.260Z+ work_dir=/work
332026-07-20T19:19:34.260Z+ input_dir=/input
342026-07-20T19:19:34.260Z+ [[ 0 == 12345 ]]
352026-07-20T19:19:34.260Z+ case "$kern" in
362026-07-20T19:19:34.260Z+ groupadd -g 12345 build
372026-07-20T19:19:34.260Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382026-07-20T19:19:36.265Z+ zfs create -o mountpoint=/work rpool/work
392026-07-20T19:19:36.587Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402026-07-20T19:19:36.592Z+ home_fs=zfs
412026-07-20T19:19:36.592Z+ [[ zfs == autofs ]]
422026-07-20T19:19:36.592Z+ mkdir -p /home/build
432026-07-20T19:19:36.592Z+ chown build:build /home/build /work
442026-07-20T19:19:38.589Z+ chmod 0700 /home/build /work
452026-07-20T19:19:38.594Zprocess exited: duration 4340 ms, exit code 0
 
462026-07-20T19:19:38.606Zstarting task 1: "authentication"
472026-07-20T19:19:38.625Zprocess exited: duration 19 ms, exit code 0
 
482026-07-20T19:19:38.635Zstarting task 2: "build"
492026-07-20T19:19:38.639Z+ banner cores
502026-07-20T19:19:38.649Z
512026-07-20T19:19:38.649Z #### #### ##### ###### ####
522026-07-20T19:19:38.649Z # # # # # # # #
532026-07-20T19:19:38.649Z # # # # # ##### ####
542026-07-20T19:19:38.649Z # # # ##### # #
552026-07-20T19:19:38.649Z # # # # # # # # #
562026-07-20T19:19:38.649Z #### #### # # ###### ####
572026-07-20T19:19:38.649Z
582026-07-20T19:19:38.649Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
592026-07-20T19:19:38.653Z+ echo 'input bins dir contains:'
602026-07-20T19:19:38.653Zinput bins dir contains:
612026-07-20T19:19:38.653Z+ ls -ltr /input/build/work/bins
622026-07-20T19:19:38.654Ztotal 1014838
632026-07-20T19:19:38.654Z-rw-r--r-- 1 root root 152760145 Jul 20 19:19 crucible-downstairs.gz
642026-07-20T19:19:38.654Z-rw-r--r-- 1 root root 145722369 Jul 20 19:19 crucible-hammer.gz
652026-07-20T19:19:38.654Z-rw-r--r-- 1 root root 151806302 Jul 20 19:19 crutest.gz
662026-07-20T19:19:38.654Z-rw-r--r-- 1 root root 68966502 Jul 20 19:19 dsc.gz
672026-07-20T19:19:38.654Z+ banner unpack
682026-07-20T19:19:38.654Z
692026-07-20T19:19:38.654Z # # # # ##### ## #### # #
702026-07-20T19:19:38.654Z # # ## # # # # # # # # #
712026-07-20T19:19:38.654Z # # # # # # # # # # ####
722026-07-20T19:19:38.654Z # # # # # ##### ###### # # #
732026-07-20T19:19:38.654Z # # # ## # # # # # # #
742026-07-20T19:19:38.654Z #### # # # # # #### # #
752026-07-20T19:19:38.654Z
762026-07-20T19:19:38.654Z+ mkdir -p /var/tmp/bins
772026-07-20T19:19:38.655Z+ for t in "$input/bins/"*.gz
782026-07-20T19:19:38.655Z++ basename /input/build/work/bins/crucible-downstairs.gz
792026-07-20T19:19:38.655Z+ b=crucible-downstairs.gz
802026-07-20T19:19:38.655Z+ b=crucible-downstairs
812026-07-20T19:19:38.655Z+ gunzip
822026-07-20T19:19:41.226Z+ chmod +x /var/tmp/bins/crucible-downstairs
832026-07-20T19:19:41.229Z+ for t in "$input/bins/"*.gz
842026-07-20T19:19:41.229Z++ basename /input/build/work/bins/crucible-hammer.gz
852026-07-20T19:19:41.230Z+ b=crucible-hammer.gz
862026-07-20T19:19:41.230Z+ b=crucible-hammer
872026-07-20T19:19:41.230Z+ gunzip
882026-07-20T19:19:43.636Z+ chmod +x /var/tmp/bins/crucible-hammer
892026-07-20T19:19:43.641Z+ for t in "$input/bins/"*.gz
902026-07-20T19:19:43.641Z++ basename /input/build/work/bins/crutest.gz
912026-07-20T19:19:43.641Z+ b=crutest.gz
922026-07-20T19:19:43.641Z+ b=crutest
932026-07-20T19:19:43.641Z+ gunzip
942026-07-20T19:19:46.189Z+ chmod +x /var/tmp/bins/crutest
952026-07-20T19:19:46.193Z+ for t in "$input/bins/"*.gz
962026-07-20T19:19:46.193Z++ basename /input/build/work/bins/dsc.gz
972026-07-20T19:19:46.193Z+ b=dsc.gz
982026-07-20T19:19:46.193Z+ b=dsc
992026-07-20T19:19:46.193Z+ gunzip
1002026-07-20T19:19:47.330Z+ chmod +x /var/tmp/bins/dsc
1012026-07-20T19:19:47.337Z+ export BINDIR=/var/tmp/bins
1022026-07-20T19:19:47.337Z+ BINDIR=/var/tmp/bins
1032026-07-20T19:19:47.337Z+ export RUST_BACKTRACE=1
1042026-07-20T19:19:47.337Z+ RUST_BACKTRACE=1
1052026-07-20T19:19:47.337Z+ jobpid=1088
1062026-07-20T19:19:47.337Z+ echo 'Setup debug logging'
1072026-07-20T19:19:47.337Z+ mkdir /tmp/debug
1082026-07-20T19:19:47.337ZSetup debug logging
1092026-07-20T19:19:47.337Z+ sleep 7200
1102026-07-20T19:19:47.337Z+ psrinfo -v
1112026-07-20T19:19:47.337Z+ df -h
1122026-07-20T19:19:47.337Z+ prstat -d d -mLc 1
1132026-07-20T19:19:47.350Z+ iostat -T d -xn 1
1142026-07-20T19:19:47.350Z+ mpstat -T d 1
1152026-07-20T19:19:47.350Z+ vmstat -T d -p 1
1162026-07-20T19:19:47.350Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1172026-07-20T19:19:47.350Z+ banner test_up_unencrypted
1182026-07-20T19:19:47.350Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1192026-07-20T19:19:47.350Z
1202026-07-20T19:19:47.351Z ##### ###### #### ##### # # ##### # # # #
1212026-07-20T19:19:47.351Z # # # # # # # # # # ## #
1222026-07-20T19:19:47.351Z # ##### #### # # # # # # # # # #
1232026-07-20T19:19:47.351Z # # # # # # ##### # # # # #
1242026-07-20T19:19:47.351Z # # # # # # # # # # # ##
1252026-07-20T19:19:47.351Z # ###### #### # ####### #### # ####### #### # #
1262026-07-20T19:19:47.351Z
1272026-07-20T19:19:47.351Z+ ptime -m bash /input/build/work/scripts/test_up.sh -N unencrypted
1282026-07-20T19:19:47.352Z/input/build/work
1292026-07-20T19:19:47.358ZTurn off color for downstairs dump
1302026-07-20T19:19:47.369Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1312026-07-20T19:19:47.369ZCreating 3 downstairs regions
1322026-07-20T19:19:47.583ZStarting 3 downstairs
1332026-07-20T19:19:47.591Zdsc started at PID: 1133
1342026-07-20T19:19:52.631ZDisable automatic restart on all downstairs
1352026-07-20T19:19:52.653Z
1362026-07-20T19:19:52.653ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1372026-07-20T19:19:52.657ZRunning test: span
1382026-07-20T19:19:56.735ZCompleted test: span
1392026-07-20T19:20:05.077ZRunning test: big
1402026-07-20T19:20:05.789ZCompleted test: big
1412026-07-20T19:20:10.796ZRunning test: dep
1422026-07-20T19:20:18.324ZCompleted test: dep
1432026-07-20T19:20:23.328ZRunning test: balloon
1442026-07-20T19:20:29.820ZCompleted test: balloon
1452026-07-20T19:20:34.823ZRunning test: deactivate
1462026-07-20T19:20:38.905ZCompleted test: deactivate
1472026-07-20T19:20:43.910ZRunning hammer
1482026-07-20T19:20:45.112ZRun repair tests
1492026-07-20T19:20:45.116Z/var/tmp/bins/crutest fill -g 31 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1502026-07-20T19:20:45.133Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.075282363Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"task":"crutest"}
1512026-07-20T19:20:45.138Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.075597635Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"task":"crutest"}
1522026-07-20T19:20:45.138Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.076166722Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"task":"crutest"}
1532026-07-20T19:20:45.138Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.076420387Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170}
1542026-07-20T19:20:45.138Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3ecc16dae322dc64f81e40811af0ed148a9fbca0\",\n git_commit_timestamp: \"2026-07-20T19:05:26.000000000Z\",\n git_branch: \"alan/printext\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.076451825Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170}
1552026-07-20T19:20:45.138Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.076460945Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170}
1562026-07-20T19:20:45.138Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.076473554Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170}
1572026-07-20T19:20:45.138Z{"msg":"Crucible 75a213f8-dea2-4a79-8ee3-addd1d3af094 has session id: 8b15bfe1-7b18-467d-800c-a177932b517e","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.076789176Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1582026-07-20T19:20:45.138Z{"msg":"Upstairs opts: Upstairs UUID: 75a213f8-dea2-4a79-8ee3-addd1d3af094, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.076816234Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1592026-07-20T19:20:45.138Z{"msg":"Crucible stats registered with UUID: 75a213f8-dea2-4a79-8ee3-addd1d3af094","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.076826924Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1602026-07-20T19:20:45.139Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-20T19:20:44.076835983Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1612026-07-20T19:20:45.139Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.076966506Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"":"io task","client":"0","":"downstairs","session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1622026-07-20T19:20:45.139Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.076985614Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"":"io task","client":"1","":"downstairs","session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1632026-07-20T19:20:45.139Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.076996684Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"":"io task","client":"2","":"downstairs","session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1642026-07-20T19:20:45.139Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.077197932Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"task":"crutest"}
1652026-07-20T19:20:45.139Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.077211141Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170}
1662026-07-20T19:20:45.139Z{"msg":"75a213f8-dea2-4a79-8ee3-addd1d3af094 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.07723181Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1672026-07-20T19:20:45.139Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.07724153Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"client":"0","":"downstairs","session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1682026-07-20T19:20:45.139Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.077252049Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"client":"1","":"downstairs","session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1692026-07-20T19:20:45.139Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.077266548Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"client":"2","":"downstairs","session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1702026-07-20T19:20:45.139Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.077276508Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"":"io task","client":"2","":"downstairs","session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1712026-07-20T19:20:45.139Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.077287047Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"":"io task","client":"0","":"downstairs","session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1722026-07-20T19:20:45.140Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.077301526Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"":"io task","client":"1","":"downstairs","session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1732026-07-20T19:20:45.140Z{"msg":"ds_connection connected from Ok(127.0.0.1:62382)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.077347343Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"":"io task","client":"0","":"downstairs","session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1742026-07-20T19:20:45.140Z{"msg":"ds_connection connected from Ok(127.0.0.1:58159)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.077362713Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"":"io task","client":"2","":"downstairs","session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1752026-07-20T19:20:45.140Z{"msg":"ds_connection connected from Ok(127.0.0.1:41812)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.077373192Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"":"io task","client":"1","":"downstairs","session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1762026-07-20T19:20:45.140Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.078011475Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"client":"0","":"downstairs","session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1772026-07-20T19:20:45.140Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.078038513Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"client":"1","":"downstairs","session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1782026-07-20T19:20:45.140Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.078049753Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"client":"2","":"downstairs","session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1792026-07-20T19:20:45.140Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.078186725Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1802026-07-20T19:20:45.140Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.078206214Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1812026-07-20T19:20:45.140Z{"msg":"[0]R flush_numbers: [1495, 1495, 1495, 1495, 1495]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.078260671Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"":"downstairs","":"downstairs","session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1822026-07-20T19:20:45.140Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.07827179Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"":"downstairs","":"downstairs","session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1832026-07-20T19:20:45.140Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.078282649Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"":"downstairs","":"downstairs","session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1842026-07-20T19:20:45.140Z{"msg":"[1]R flush_numbers: [1495, 1495, 1495, 1495, 1495]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.078293319Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"":"downstairs","":"downstairs","session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1852026-07-20T19:20:45.140Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.078303568Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"":"downstairs","":"downstairs","session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1862026-07-20T19:20:45.140Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.078314147Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"":"downstairs","":"downstairs","session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1872026-07-20T19:20:45.140Z{"msg":"[2]R flush_numbers: [1495, 1495, 1495, 1495, 1495]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.078333356Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"":"downstairs","":"downstairs","session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1882026-07-20T19:20:45.140Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.078343996Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"":"downstairs","":"downstairs","session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1892026-07-20T19:20:45.141Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.078354725Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"":"downstairs","":"downstairs","session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1902026-07-20T19:20:45.141Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.078364945Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"":"downstairs","":"downstairs","session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1912026-07-20T19:20:45.141Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.078375144Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"":"downstairs","":"downstairs","session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1922026-07-20T19:20:45.141Z{"msg":"Next flush: 1496","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.078384563Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"":"downstairs","":"downstairs","session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1932026-07-20T19:20:45.141Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.078394353Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"":"downstairs","":"downstairs","session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1942026-07-20T19:20:45.141Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.078404502Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1952026-07-20T19:20:45.141Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.078414022Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"client":"0","":"downstairs","session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1962026-07-20T19:20:45.141Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.078424761Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"client":"1","":"downstairs","session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1972026-07-20T19:20:45.141Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.07843525Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"client":"2","":"downstairs","session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1982026-07-20T19:20:45.141Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.07844558Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
1992026-07-20T19:20:45.141Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.078455499Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
2002026-07-20T19:20:45.141Z{"msg":"75a213f8-dea2-4a79-8ee3-addd1d3af094 is now active with session: 8b15bfe1-7b18-467d-800c-a177932b517e","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.078466339Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
2012026-07-20T19:20:45.141Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.078475538Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"session_id":"8b15bfe1-7b18-467d-800c-a177932b517e"}
2022026-07-20T19:20:45.141Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.078485138Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170}
2032026-07-20T19:20:45.141Z{"msg":"Activated sub_volume 75a213f8-dea2-4a79-8ee3-addd1d3af094","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.078493537Z","hostname":"Disk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2042026-07-20T19:20:45.141Zw-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170}
2052026-07-20T19:20:45.141Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.078507306Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"task":"crutest"}
2062026-07-20T19:20:45.141Z{"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.078533485Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"task":"crutest"}
2072026-07-20T19:20:45.145ZRead and Verify all blocks (0..50 range:false)
2082026-07-20T19:20:45.145Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30----------------------------------------------------------------
2092026-07-20T19:20:45.145Z,"time":"2026-07-20T19:20:44.084418914Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R" Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2102026-07-20T19:20:45.145Z,"States:pid": Active1170 Active Active
2112026-07-20T19:20:45.145Z,"JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2122026-07-20T19:20:45.145Ztask":"crutest"}
2132026-07-20T19:20:45.145Z 1002 Acked Read 50{ Done" Done Donemsg" false:
2142026-07-20T19:20:45.145Z" STATES DS:0 DS:1 DS:2 TOTAL
2152026-07-20T19:20:45.145ZCLIENT: Tests done. All submitted work has been ACK'd Sent "," 0 v" 0 : 0 0 0
2162026-07-20T19:20:45.145Z," Done name 1 ": 1 " 1 3
2172026-07-20T19:20:45.145Zcrucible Skipped " 0 ," 0 level 0 " 0
2182026-07-20T19:20:45.145Z:30 Error 0 0 0 0
2192026-07-20T19:20:45.145ZLast Flush: Some(JobId(1001)) ,Some(JobId(1001)) "Some(JobId(1001)) time"
2202026-07-20T19:20:45.145Z:Downstairs last five completed:"2026-07-20T19:20:44.084459082Z"," 1001hostname" 1000:"
2212026-07-20T19:20:45.145Zw-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,Upstairs last five completed: "task 1002": 1001" 1000
2222026-07-20T19:20:45.145Zcrutest"}
2232026-07-20T19:20:45.145Z{"msg":"CLIENT: Up:0 ds:1 act:3","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:44.08448707Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"task":"crutest"}
2242026-07-20T19:20:49.144Z----------------------------------------------------------------
2252026-07-20T19:20:49.145Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2262026-07-20T19:20:49.145ZDownstairs last five completed: 1003 1002 1001 1000
2272026-07-20T19:20:49.145ZUpstairs last five completed: 1003 1002 1001 1000
2282026-07-20T19:20:49.149Z{"msg":"CLIENT: Up:0 ds:0 act:3","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.086497609Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"task":"crutest"}
2292026-07-20T19:20:49.149Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.086544307Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170,"task":"crutest"}
2302026-07-20T19:20:49.149Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-20T19:20:48.086555476Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1170}
2312026-07-20T19:20:49.153ZRepair setup passed
2322026-07-20T19:20:49.156ZCopy the region for /var/tmp/test_up-build/8830
2332026-07-20T19:20:49.156Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2342026-07-20T19:20:49.156Z/var/tmp/bins/crutest fill -g 32 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2352026-07-20T19:20:49.174Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.116241516Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"task":"crutest"}
2362026-07-20T19:20:49.179Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.116552618Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"task":"crutest"}
2372026-07-20T19:20:49.179Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.117260217Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"task":"crutest"}
2382026-07-20T19:20:49.179Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.117495714Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176}
2392026-07-20T19:20:49.179Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3ecc16dae322dc64f81e40811af0ed148a9fbca0\",\n git_commit_timestamp: \"2026-07-20T19:05:26.000000000Z\",\n git_branch: \"alan/printext\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.117522202Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176}
2402026-07-20T19:20:49.179Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.117535311Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176}
2412026-07-20T19:20:49.179Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.117547581Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176}
2422026-07-20T19:20:49.179Z{"msg":"Crucible b802c0fc-346c-4943-9015-634d2417f7ef has session id: 59a6678d-78ac-47cd-97b5-9dc329dc9893","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.11790979Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2432026-07-20T19:20:49.179Z{"msg":"Upstairs opts: Upstairs UUID: b802c0fc-346c-4943-9015-634d2417f7ef, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.117935468Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2442026-07-20T19:20:49.180Z{"msg":"Crucible stats registered with UUID: b802c0fc-346c-4943-9015-634d2417f7ef","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.117945878Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2452026-07-20T19:20:49.180Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-20T19:20:48.117961147Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2462026-07-20T19:20:49.180Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.118139196Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"":"io task","client":"2","":"downstairs","session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2472026-07-20T19:20:49.180Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.118176464Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"":"io task","client":"0","":"downstairs","session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2482026-07-20T19:20:49.180Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.118187574Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"":"io task","client":"1","":"downstairs","session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2492026-07-20T19:20:49.180Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.118317836Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"task":"crutest"}
2502026-07-20T19:20:49.180Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.118332025Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176}
2512026-07-20T19:20:49.180Z{"msg":"b802c0fc-346c-4943-9015-634d2417f7ef active request set","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.118375473Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2522026-07-20T19:20:49.180Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.118385852Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"client":"0","":"downstairs","session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2532026-07-20T19:20:49.180Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.118400461Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"client":"1","":"downstairs","session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2542026-07-20T19:20:49.180Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.118410121Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"client":"2","":"downstairs","session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2552026-07-20T19:20:49.180Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.11842047Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"":"io task","client":"2","":"downstairs","session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2562026-07-20T19:20:49.180Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.118430679Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"":"io task","client":"0","":"downstairs","session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2572026-07-20T19:20:49.180Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.118441249Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"":"io task","client":"1","":"downstairs","session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2582026-07-20T19:20:49.180Z{"msg":"ds_connection connected from Ok(127.0.0.1:56916)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.118456858Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"":"io task","client":"2","":"downstairs","session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2592026-07-20T19:20:49.180Z{"msg":"ds_connection connected from Ok(127.0.0.1:40138)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.118499815Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"":"io task","client":"0","":"downstairs","session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2602026-07-20T19:20:49.180Z{"msg":"ds_connection connected from Ok(127.0.0.1:61296)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.118540063Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"":"io task","client":"1","":"downstairs","session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2612026-07-20T19:20:49.180Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.11912031Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"client":"1","":"downstairs","session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2622026-07-20T19:20:49.180Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.119162997Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"client":"2","":"downstairs","session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2632026-07-20T19:20:49.180Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.119186596Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"client":"0","":"downstairs","session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2642026-07-20T19:20:49.180Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.119342297Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2652026-07-20T19:20:49.180Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.119356556Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2662026-07-20T19:20:49.180Z{"msg":"[0]R flush_numbers: [1496, 1496, 1496, 1496, 1496]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.119376045Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"":"downstairs","":"downstairs","session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2672026-07-20T19:20:49.180Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.119390874Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"":"downstairs","":"downstairs","session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2682026-07-20T19:20:49.180Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.119401303Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"":"downstairs","":"downstairs","session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2692026-07-20T19:20:49.180Z{"msg":"[1]R flush_numbers: [1496, 1496, 1496, 1496, 1496]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.119415982Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"":"downstairs","":"downstairs","session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2702026-07-20T19:20:49.180Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.119425852Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"":"downstairs","":"downstairs","session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2712026-07-20T19:20:49.180Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.119439921Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"":"downstairs","":"downstairs","session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2722026-07-20T19:20:49.180Z{"msg":"[2]R flush_numbers: [1496, 1496, 1496, 1496, 1496]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.11945027Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"":"downstairs","":"downstairs","session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2732026-07-20T19:20:49.180Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.11946443Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"":"downstairs","":"downstairs","session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2742026-07-20T19:20:49.180Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.119474379Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"":"downstairs","":"downstairs","session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2752026-07-20T19:20:49.180Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.119488098Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"":"downstairs","":"downstairs","session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2762026-07-20T19:20:49.180Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.119497908Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"":"downstairs","":"downstairs","session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2772026-07-20T19:20:49.180Z{"msg":"Next flush: 1497","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.119515197Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"":"downstairs","":"downstairs","session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2782026-07-20T19:20:49.181Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.119524886Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"":"downstairs","":"downstairs","session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2792026-07-20T19:20:49.181Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.119539035Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2802026-07-20T19:20:49.181Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.119547785Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"client":"0","":"downstairs","session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2812026-07-20T19:20:49.181Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.119561774Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"client":"1","":"downstairs","session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2822026-07-20T19:20:49.181Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.119571283Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"client":"2","":"downstairs","session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2832026-07-20T19:20:49.181Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.119585543Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2842026-07-20T19:20:49.181Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.119594552Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2852026-07-20T19:20:49.181Z{"msg":"b802c0fc-346c-4943-9015-634d2417f7ef is now active with session: 59a6678d-78ac-47cd-97b5-9dc329dc9893","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.119612321Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2862026-07-20T19:20:49.181Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.119621011Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"session_id":"59a6678d-78ac-47cd-97b5-9dc329dc9893"}
2872026-07-20T19:20:49.181Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.11963462Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176}
2882026-07-20T19:20:49.181Z{"msg":"Activated sub_volume b802c0fc-346c-4943-9015-634d2417f7ef","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2892026-07-20T19:20:49.181Z,"time":"2026-07-20T19:20:48.119643159Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176}
2902026-07-20T19:20:49.181Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.119655319Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"task":"crutest"}
2912026-07-20T19:20:49.181ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
2922026-07-20T19:20:49.181Z{"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.119985919Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"task":"crutest"}
2932026-07-20T19:20:49.187ZRead and Verify all blocks (0..50 range:false)
2942026-07-20T19:20:49.187Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.125580405Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"task":"crutest"}
2952026-07-20T19:20:49.187Z{"msg":"CLIENT: Tests done. All submitted work has been ACK'd","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.125591245Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"task":"crutest"}
2962026-07-20T19:20:49.187Z----------------------------------------------------------------
2972026-07-20T19:20:49.187Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
2982026-07-20T19:20:49.187ZStates: Active Active Active
2992026-07-20T19:20:49.187ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
3002026-07-20T19:20:49.187Z 1002 Acked Read 50 Done Done Done false
3012026-07-20T19:20:49.187Z STATES DS:0 DS:1 DS:2 TOTAL
3022026-07-20T19:20:49.187Z Sent 0 0 0 0
3032026-07-20T19:20:49.187Z Done 1 1 1 3
3042026-07-20T19:20:49.187Z Skipped 0 0 0 0
3052026-07-20T19:20:49.187Z Error 0 0 0 0
3062026-07-20T19:20:49.187ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
3072026-07-20T19:20:49.187ZDownstairs last five completed: 1001 1000
3082026-07-20T19:20:49.187ZUpstairs last five completed: 1002 1001 1000
3092026-07-20T19:20:49.187Z{"msg":"CLIENT: Up:0 ds:1 act:3","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:48.125689599Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"task":"crutest"}
3102026-07-20T19:20:53.187Z----------------------------------------------------------------
3112026-07-20T19:20:53.187Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
3122026-07-20T19:20:53.187ZDownstairs last five completed: 1003 1002 1001 1000
3132026-07-20T19:20:53.187ZUpstairs last five completed: 1003 1002 1001 1000
3142026-07-20T19:20:53.192Z{"msg":"CLIENT: Up:0 ds:0 act:3","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:52.127625723Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"task":"crutest"}
3152026-07-20T19:20:53.192Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:52.1276766Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1176,"task":"crutest"}
3162026-07-20T19:20:53.192ZRepair part 1 passed
3172026-07-20T19:20:53.192Z
3182026-07-20T19:20:53.192ZKill the current downstairs
3192026-07-20T19:20:53.233ZDownstairs 2 stopped
3202026-07-20T19:20:53.233Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
3212026-07-20T19:20:53.239ZNow put back the original so we have a mismatch
3222026-07-20T19:20:53.239Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
3232026-07-20T19:20:53.239ZRestart downstairs with old directory
3242026-07-20T19:20:53.248Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
3252026-07-20T19:20:53.323ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3262026-07-20T19:20:53.329Z 0 000-009 32 32 31 1497 1497 1496 F F F <---
3272026-07-20T19:20:53.329Z 1 010-019 32 32 31 1497 1497 1496 F F F <---
3282026-07-20T19:20:53.329Z 2 020-029 32 32 31 1497 1497 1496 F F F <---
3292026-07-20T19:20:53.329Z 3 030-039 32 32 31 1497 1497 1496 F F F <---
3302026-07-20T19:20:53.329Z 4 040-049 32 32 31 1497 1497 1496 F F F <---
3312026-07-20T19:20:53.330ZMax gen: 32, Max flush: 1497
3322026-07-20T19:20:53.470ZError: Difference in extent metadata found!
3332026-07-20T19:20:53.470Z
3342026-07-20T19:20:53.470ZStack backtrace:
3352026-07-20T19:20:53.470Z 0: anyhow::error::<impl anyhow::Error>::msg
3362026-07-20T19:20:53.470Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14
3372026-07-20T19:20:53.470Z 1: anyhow::__private::format_err
3382026-07-20T19:20:53.470Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13
3392026-07-20T19:20:53.488Zdump test found error as expected
3402026-07-20T19:20:53.488Z
3412026-07-20T19:20:53.488Z
3422026-07-20T19:20:53.492Z/var/tmp/bins/crutest verify --range -g 33 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
3432026-07-20T19:20:53.506Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:52.447727091Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"task":"crutest"}
3442026-07-20T19:20:53.510Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:52.448051432Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"task":"crutest"}
3452026-07-20T19:20:53.510Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:52.448856155Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"task":"crutest"}
3462026-07-20T19:20:53.510Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:52.449091552Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184}
3472026-07-20T19:20:53.510Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3ecc16dae322dc64f81e40811af0ed148a9fbca0\",\n git_commit_timestamp: \"2026-07-20T19:05:26.000000000Z\",\n git_branch: \"alan/printext\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:52.44912142Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184}
3482026-07-20T19:20:53.510Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:52.449135039Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184}
3492026-07-20T19:20:53.510Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:52.449147409Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184}
3502026-07-20T19:20:53.510Z{"msg":"Crucible 3b39ba62-63d9-4f3a-8532-4c1e7d870cf5 has session id: 7a93459d-6598-479b-9b8f-4371de37d7fd","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:52.449538536Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3512026-07-20T19:20:53.510Z{"msg":"Upstairs opts: Upstairs UUID: 3b39ba62-63d9-4f3a-8532-4c1e7d870cf5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:52.449566764Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3522026-07-20T19:20:53.510Z{"msg":"Crucible stats registered with UUID: 3b39ba62-63d9-4f3a-8532-4c1e7d870cf5","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:52.449581543Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3532026-07-20T19:20:53.510Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-20T19:20:52.449590803Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3542026-07-20T19:20:53.510Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:52.449711136Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"io task","client":"1","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3552026-07-20T19:20:53.510Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:52.449733685Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"io task","client":"0","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3562026-07-20T19:20:53.510Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:52.449753553Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"io task","client":"2","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3572026-07-20T19:20:53.511Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:52.449953812Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"task":"crutest"}
3582026-07-20T19:20:53.511Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:52.44998074Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184}
3592026-07-20T19:20:53.511Z{"msg":"3b39ba62-63d9-4f3a-8532-4c1e7d870cf5 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:52.450052636Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3602026-07-20T19:20:53.511Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:52.450066745Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"client":"0","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3612026-07-20T19:20:53.511Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:52.450076895Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"client":"1","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3622026-07-20T19:20:53.511Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:52.450090144Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"client":"2","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3632026-07-20T19:20:53.511Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:52.450099873Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"io task","client":"0","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3642026-07-20T19:20:53.511Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:52.450129482Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"io task","client":"2","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3652026-07-20T19:20:53.511Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:52.45016147Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"io task","client":"1","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3662026-07-20T19:20:53.511Z{"msg":"ds_connection connected from Ok(127.0.0.1:57043)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:52.450190118Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"io task","client":"0","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3672026-07-20T19:20:53.512Z{"msg":"ds_connection connected from Ok(127.0.0.1:41875)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:52.450206477Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"io task","client":"1","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3682026-07-20T19:20:53.512Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-07-20T19:20:52.450299262Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"io task","client":"2","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3692026-07-20T19:20:53.512Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-07-20T19:20:52.45033084Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"io task","client":"2","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3702026-07-20T19:20:53.512Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-07-20T19:20:52.450344269Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3712026-07-20T19:20:53.512Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:52.450360728Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"io task","client":"2","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3722026-07-20T19:20:53.512Z{"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:52.450410875Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"io task","client":"2","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3732026-07-20T19:20:53.512Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:52.450918936Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"client":"0","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3742026-07-20T19:20:53.512Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:52.450958184Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"client":"1","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3752026-07-20T19:20:53.512Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:52.451120954Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3762026-07-20T19:20:53.512Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:20:52.451135093Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3772026-07-20T19:21:03.511Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.451859751Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"io task","client":"2","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3782026-07-20T19:21:03.517Z{"msg":"ds_connection connected from Ok(127.0.0.1:53161)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.452003463Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"io task","client":"2","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3792026-07-20T19:21:03.517Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.452661645Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"client":"2","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3802026-07-20T19:21:03.517Z{"msg":"[0]R flush_numbers: [1497, 1497, 1497, 1497, 1497]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.452839325Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"downstairs","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3812026-07-20T19:21:03.517Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.452855544Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"downstairs","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3822026-07-20T19:21:03.517Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.452865953Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"downstairs","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3832026-07-20T19:21:03.517Z{"msg":"[1]R flush_numbers: [1497, 1497, 1497, 1497, 1497]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.452880642Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"downstairs","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3842026-07-20T19:21:03.517Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.452890732Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"downstairs","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3852026-07-20T19:21:03.517Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.452900911Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"downstairs","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3862026-07-20T19:21:03.517Z{"msg":"[2]R flush_numbers: [1496, 1496, 1496, 1496, 1496]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.45291537Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"downstairs","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3872026-07-20T19:21:03.517Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.45292539Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"downstairs","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3882026-07-20T19:21:03.517Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.452939559Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"downstairs","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3892026-07-20T19:21:03.517Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.452949238Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"downstairs","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3902026-07-20T19:21:03.517Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.452963147Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"downstairs","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3912026-07-20T19:21:03.517Z{"msg":"Next flush: 1498","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.452972477Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"downstairs","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3922026-07-20T19:21:03.517Z{"msg":"extent 0 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1497, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1497, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1496, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.452991556Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"mend","":"downstairs","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3932026-07-20T19:21:03.517Z{"msg":"extent:0 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1497, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1497, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1496, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.453022274Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"mend","":"downstairs","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3942026-07-20T19:21:03.518Z{"msg":"found dest for source 0 for extent at index 0 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.453037243Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"mend","":"downstairs","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3952026-07-20T19:21:03.518Z{"msg":"extent 1 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1497, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1497, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1496, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.453051552Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"mend","":"downstairs","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3962026-07-20T19:21:03.518Z{"msg":"extent:1 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1497, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1497, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1496, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.453069931Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"mend","":"downstairs","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3972026-07-20T19:21:03.518Z{"msg":"found dest for source 0 for extent at index 1 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.453081501Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"mend","":"downstairs","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3982026-07-20T19:21:03.518Z{"msg":"extent 2 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1497, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1497, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1496, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.45310003Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"mend","":"downstairs","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
3992026-07-20T19:21:03.518Z{"msg":"extent:2 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1497, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1497, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1496, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.453118418Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"mend","":"downstairs","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
4002026-07-20T19:21:03.518Z{"msg":"found dest for source 0 for extent at index 2 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.453138087Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"mend","":"downstairs","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
4012026-07-20T19:21:03.518Z{"msg":"extent 3 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1497, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1497, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1496, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.453152407Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"mend","":"downstairs","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
4022026-07-20T19:21:03.518Z{"msg":"extent:3 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1497, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1497, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1496, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.453170815Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"mend","":"downstairs","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
4032026-07-20T19:21:03.518Z{"msg":"found dest for source 0 for extent at index 3 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.453181745Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"mend","":"downstairs","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
4042026-07-20T19:21:03.518Z{"msg":"extent 4 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1497, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1497, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1496, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.453200234Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"mend","":"downstairs","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
4052026-07-20T19:21:03.519Z{"msg":"extent:4 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1497, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1497, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1496, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.453214603Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"mend","":"downstairs","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
4062026-07-20T19:21:03.519Z{"msg":"found dest for source 0 for extent at index 4 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.453229442Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"mend","":"downstairs","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
4072026-07-20T19:21:03.519Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.453244351Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"client":"0","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
4082026-07-20T19:21:03.519Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.45325956Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"client":"1","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
4092026-07-20T19:21:03.519Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.45326977Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"client":"2","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
4102026-07-20T19:21:03.519Z{"msg":"Full repair list: {ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.453290019Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"downstairs","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
4112026-07-20T19:21:03.519Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 1498, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 1498, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 1498, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 1498, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 1498, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.453446409Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"downstairs","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
4122026-07-20T19:21:03.519Z{"msg":"starting reconciliation 49967cf0-63ee-409e-aca4-ba043b1b2268: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.453458529Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"downstairs","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
4132026-07-20T19:21:03.519Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.453468468Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"client":"2","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
4142026-07-20T19:21:03.522Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.463817309Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"client":"2","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
4152026-07-20T19:21:03.526Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.465875Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"client":"2","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
4162026-07-20T19:21:03.529Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.467872874Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"client":"2","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
4172026-07-20T19:21:03.529Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.469720867Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"client":"2","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
4182026-07-20T19:21:03.532Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.471404529Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"":"downstairs","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
4192026-07-20T19:21:03.532Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.471427098Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"client":"0","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
4202026-07-20T19:21:03.532Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.471437727Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"client":"1","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
4212026-07-20T19:21:03.532Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.471452076Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"client":"2","":"downstairs","session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
4222026-07-20T19:21:03.532Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.471462216Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
4232026-07-20T19:21:03.532Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.471471455Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
4242026-07-20T19:21:03.532Z{"msg":"3b39ba62-63d9-4f3a-8532-4c1e7d870cf5 is now active with session: 7a93459d-6598-479b-9b8f-4371de37d7fd","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.471485955Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"session_id":"7a93459d-6598-479b-9b8f-4371de37d7fd"}
4252026-07-20T19:21:03.532Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.471495294Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184}
4262026-07-20T19:21:03.532Z{"msg":"Activated sub_volume 3b39ba62-63d9-4f3a-8532-4c1e7d870cf5","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.471508173Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184}
4272026-07-20T19:21:03.532Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.47157Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"task":"crutest"}
4282026-07-20T19:21:03.532ZDisk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
4292026-07-20T19:21:03.532ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4302026-07-20T19:21:03.532ZRead and Verify all blocks (0..50 range:true)
4312026-07-20T19:21:03.536Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.475295964Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"task":"crutest"}
4322026-07-20T19:21:03.536Z{"msg":"Verify test completed","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.475322532Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"task":"crutest"}
4332026-07-20T19:21:03.536Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.475332522Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"task":"crutest"}
4342026-07-20T19:21:03.536Z{"msg":"CLIENT: Tests done. All submitted work has been ACK'd","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.475345951Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"task":"crutest"}
4352026-07-20T19:21:03.536Z----------------------------------------------------------------
4362026-07-20T19:21:03.536Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
4372026-07-20T19:21:03.536ZStates: Active Active Active
4382026-07-20T19:21:03.536ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4392026-07-20T19:21:03.536Z 1000 Acked Read 50 Done Done Done false
4402026-07-20T19:21:03.536Z STATES DS:0 DS:1 DS:2 TOTAL
4412026-07-20T19:21:03.536Z Sent 0 0 0 0
4422026-07-20T19:21:03.536Z Done 1 1 1 3
4432026-07-20T19:21:03.536Z Skipped 0 0 0 0
4442026-07-20T19:21:03.536Z Error 0 0 0 0
4452026-07-20T19:21:03.536ZLast Flush: None None None
4462026-07-20T19:21:03.536ZDownstairs last five completed:
4472026-07-20T19:21:03.536ZUpstairs last five completed: 1000
4482026-07-20T19:21:03.536Z{"msg":"CLIENT: Up:0 ds:1 act:3","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:02.475430696Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"task":"crutest"}
4492026-07-20T19:21:07.535Z----------------------------------------------------------------
4502026-07-20T19:21:07.535Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
4512026-07-20T19:21:07.535ZDownstairs last five completed: 1001 1000
4522026-07-20T19:21:07.535ZUpstairs last five completed: 1001 1000
4532026-07-20T19:21:07.539Z{"msg":"CLIENT: Up:0 ds:0 act:3","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:06.47632083Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"task":"crutest"}
4542026-07-20T19:21:07.539Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-07-20T19:21:06.476347889Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184,"task":"crutest"}
4552026-07-20T19:21:07.539Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-20T19:21:06.476357798Z","hostname":"w-01KY0FEA4FNRZ5VB7NM1FCWT0R","pid":1184}
4562026-07-20T19:21:07.542ZRepair part 2 passed
4572026-07-20T19:21:07.542Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4582026-07-20T19:21:07.552ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4592026-07-20T19:21:07.552Z 0 000-009 32 32 32 1497 1497 1497 F F F
4602026-07-20T19:21:07.552Z 1 010-019 32 32 32 1497 1497 1497 F F F
4612026-07-20T19:21:07.555Z 2 020-029 32 32 32 1497 1497 1497 F F F
4622026-07-20T19:21:07.555Z 3 030-039 32 32 32 1497 1497 1497 F F F
4632026-07-20T19:21:07.555Z 4 040-049 32 32 32 1497 1497 1497 F F F
4642026-07-20T19:21:07.555ZMax gen: 32, Max flush: 1497
4652026-07-20T19:21:07.556Zdump test passed
4662026-07-20T19:21:07.556Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
4672026-07-20T19:21:07.565Z Extent 1
4682026-07-20T19:21:07.568ZGEN 32 32 32
4692026-07-20T19:21:07.568ZFLUSH_ID 1497 1497 1497
4702026-07-20T19:21:07.569ZDIRTY
4712026-07-20T19:21:07.569Z
4722026-07-20T19:21:07.569ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
4732026-07-20T19:21:07.569Z010 A A A A A A
4742026-07-20T19:21:07.569Z011 A A A A A A
4752026-07-20T19:21:07.569Z012 A A A A A A
4762026-07-20T19:21:07.569Z013 A A A A A A
4772026-07-20T19:21:07.572Z014 A A A A A A
4782026-07-20T19:21:07.572Z015 A A A A A A
4792026-07-20T19:21:07.572Z016 A A A A A A
4802026-07-20T19:21:07.572Z017 A A A A A A
4812026-07-20T19:21:07.572Z018 A A A A A A
4822026-07-20T19:21:07.572Z019 A A A A A A
4832026-07-20T19:21:07.575Zdump extent test passed
4842026-07-20T19:21:07.575Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
4852026-07-20T19:21:07.583ZExtent 2 Block in extent 0 Actual block 20
4862026-07-20T19:21:07.583Z
4872026-07-20T19:21:07.586Z DATA SHA256 VER
4882026-07-20T19:21:07.586Z------ ---------------------------------------------------------------- ---
4892026-07-20T19:21:07.586Z 0 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
4902026-07-20T19:21:07.586Z 1 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
4912026-07-20T19:21:07.586Z 2 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
4922026-07-20T19:21:07.586Z
4932026-07-20T19:21:07.586ZNONCES 0 1 2 DIFF
4942026-07-20T19:21:07.586Z------ ------------------------ ------------------------ ------------------------ -----
4952026-07-20T19:21:07.586Z
4962026-07-20T19:21:07.587Z TAGS 0 1 2 DIFF
4972026-07-20T19:21:07.587Z------ -------------------------------- -------------------------------- -------------------------------- -----
4982026-07-20T19:21:07.587Z
4992026-07-20T19:21:07.587ZHASHES 0 1 2 DIFF
5002026-07-20T19:21:07.587Z------ ---------------- ---------------- ---------------- -----
5012026-07-20T19:21:07.587Z 0 42916ab6fa364c70 42916ab6fa364c70 42916ab6fa364c70
5022026-07-20T19:21:07.587Z
5032026-07-20T19:21:07.590Zdump block test passed
5042026-07-20T19:21:07.590ZInitial upstairs tests have completed, stopping all downstairs
5052026-07-20T19:21:12.607ZCreating 4 larger downstairs regions
5062026-07-20T19:21:12.724ZStarting 4 downstairs
5072026-07-20T19:21:17.729Zdsc restarted at PID: 1197
5082026-07-20T19:21:17.787ZNow do the replace-reconcile test
5092026-07-20T19:21:17.800ZUsing 8840 for the replacement port
5102026-07-20T19:24:39.639ZCompleted test: replace-reconcile
5112026-07-20T19:24:39.642ZNow do the replace-before-active test
5122026-07-20T19:27:27.741ZCompleted test: replace-before-active
5132026-07-20T19:27:27.741ZAll tests have completed, stopping all downstairs
5142026-07-20T19:27:27.757Z
5152026-07-20T19:27:27.777ZAll Tests have passed
5162026-07-20T19:27:27.777Z7:40 Test duration
5172026-07-20T19:27:27.777Z
5182026-07-20T19:27:27.777Zreal 7:40.386407534
5192026-07-20T19:27:27.777Zuser 40.746725470
5202026-07-20T19:27:27.777Zsys 5.037677775
5212026-07-20T19:27:27.777Ztrap 0.063068930
5222026-07-20T19:27:27.777Ztflt 0.026275810
5232026-07-20T19:27:27.777Zdflt 0.009150603
5242026-07-20T19:27:27.777Zkflt 0.001767791
5252026-07-20T19:27:27.777Zlock 4:03:19.681677255
5262026-07-20T19:27:27.778Zslp 20:18.750448114
5272026-07-20T19:27:27.778Zlat 4.116534579
5282026-07-20T19:27:27.778Zstop 0.011150507
5292026-07-20T19:27:27.778Z+ echo 'test-up-unencrypted ends'
5302026-07-20T19:27:27.778Ztest-up-unencrypted ends
5312026-07-20T19:27:36.394Zprocess exited: duration 469093 ms, exit code 0
5322026-07-20T19:27:36.394Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
5332026-07-20T19:28:35.297Zexec warning: : stdout descriptor may be held open by a background process; giving up!
5342026-07-20T19:28:35.297Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
5352026-07-20T19:28:35.385Zfound 14 output files
5362026-07-20T19:28:35.385Zuploading: /tmp/test_up-build/dsc-out.txt (46844 bytes)
5372026-07-20T19:28:36.400Zuploaded: /tmp/test_up-build/dsc-out.txt
5382026-07-20T19:28:36.400Zuploading: /tmp/test_up-build/test_up_out.txt (22518161 bytes)
5392026-07-20T19:28:37.730Zuploaded: /tmp/test_up-build/test_up_out.txt
5402026-07-20T19:28:37.730Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (2489 bytes)
5412026-07-20T19:28:38.745Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
5422026-07-20T19:28:38.745Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (548321 bytes)
5432026-07-20T19:28:39.763Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
5442026-07-20T19:28:39.764Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (978643 bytes)
5452026-07-20T19:28:40.785Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
5462026-07-20T19:28:40.785Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (543075 bytes)
5472026-07-20T19:28:41.802Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
5482026-07-20T19:28:41.802Zuploading: /tmp/debug/df.txt (1270 bytes)
5492026-07-20T19:28:42.811Zuploaded: /tmp/debug/df.txt
5502026-07-20T19:28:42.811Zuploading: /tmp/debug/dtrace.txt (504287 bytes)
5512026-07-20T19:28:43.827Zuploaded: /tmp/debug/dtrace.txt
5522026-07-20T19:28:43.827Zuploading: /tmp/debug/iostat.txt (156828 bytes)
5532026-07-20T19:28:43.835Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 156828 -> 159501
5542026-07-20T19:28:44.843Zuploaded: /tmp/debug/iostat.txt
5552026-07-20T19:28:44.843Zuploading: /tmp/debug/mpstat.txt (383525 bytes)
5562026-07-20T19:28:44.854Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 383525 -> 390785
5572026-07-20T19:28:45.863Zuploaded: /tmp/debug/mpstat.txt
5582026-07-20T19:28:45.863Zuploading: /tmp/debug/paging.txt (67510 bytes)
5592026-07-20T19:28:45.869Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 67510 -> 68900
5602026-07-20T19:28:46.877Zuploaded: /tmp/debug/paging.txt
5612026-07-20T19:28:46.877Zuploading: /tmp/debug/prstat.txt (721683 bytes)
5622026-07-20T19:28:46.889Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 721683 -> 737227
5632026-07-20T19:28:47.898Zuploaded: /tmp/debug/prstat.txt
5642026-07-20T19:28:47.899Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
5652026-07-20T19:28:48.908Zuploaded: /tmp/debug/psrinfo.txt
5662026-07-20T19:28:48.908Zuploading: /tmp/debug/upstairs-info.txt (52479 bytes)
5672026-07-20T19:28:49.921Zuploaded: /tmp/debug/upstairs-info.txt