01KVRSKNAC1S5TVE7WE8YZ3K07: test-up-unencrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01KVRSM1G71VWCCCTX34SECW4B

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12026-06-22T23:24:28.606Zjob dependencies complete; ready to run (waiting for 13 m 24 s)
22026-06-22T23:25:37.857Zjob assigned to worker 01KVRTD4RBMKHF0QR72SRTZ0MZ [factory aws, i-04dd59211d8a81d60] (queued for 1 m 9 s)
32026-06-22T23:25:45.170Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42026-06-22T23:25:47.210Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52026-06-22T23:25:47.213Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62026-06-22T23:25:48.744Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72026-06-22T23:25:48.747Zdownloading input: /input/build/work/bins/crutest.gz
82026-06-22T23:25:50.417Zdownloaded input: /input/build/work/bins/crutest.gz
92026-06-22T23:25:50.420Zdownloading input: /input/build/work/bins/dsc.gz
102026-06-22T23:25:51.167Zdownloaded input: /input/build/work/bins/dsc.gz
112026-06-22T23:25:51.171Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122026-06-22T23:25:51.200Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132026-06-22T23:25:51.205Zdownloading input: /input/build/work/scripts/test_ds.sh
142026-06-22T23:25:51.230Zdownloaded input: /input/build/work/scripts/test_ds.sh
152026-06-22T23:25:51.233Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162026-06-22T23:25:51.258Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172026-06-22T23:25:51.258Zdownloading input: /input/build/work/scripts/test_repair.sh
182026-06-22T23:25:51.287Zdownloaded input: /input/build/work/scripts/test_repair.sh
192026-06-22T23:25:51.287Zdownloading input: /input/build/work/scripts/test_replay.sh
202026-06-22T23:25:51.321Zdownloaded input: /input/build/work/scripts/test_replay.sh
212026-06-22T23:25:51.324Zdownloading input: /input/build/work/scripts/test_up.sh
222026-06-22T23:25:51.352Zdownloaded input: /input/build/work/scripts/test_up.sh
232026-06-22T23:25:51.352Zdownloading input: /input/build/work/scripts/upstairs_info.d
242026-06-22T23:25:51.382Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252026-06-22T23:25:51.386Zdownloading input: /input/build/tmp/cargo-test-out.log
262026-06-22T23:25:51.952Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272026-06-22T23:25:51.952Zstarting task 0: "setup"
282026-06-22T23:25:51.957Z++ uname -s
292026-06-22T23:25:51.961Z+ kern=SunOS
302026-06-22T23:25:51.961Z+ build_user=build
312026-06-22T23:25:51.961Z+ build_uid=12345
322026-06-22T23:25:51.961Z+ work_dir=/work
332026-06-22T23:25:51.961Z+ input_dir=/input
342026-06-22T23:25:51.961Z+ [[ 0 == 12345 ]]
352026-06-22T23:25:51.961Z+ case "$kern" in
362026-06-22T23:25:51.961Z+ groupadd -g 12345 build
372026-06-22T23:25:51.961Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382026-06-22T23:25:53.967Z+ zfs create -o mountpoint=/work rpool/work
392026-06-22T23:25:54.368Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402026-06-22T23:25:54.374Z+ home_fs=zfs
412026-06-22T23:25:54.374Z+ [[ zfs == autofs ]]
422026-06-22T23:25:54.374Z+ mkdir -p /home/build
432026-06-22T23:25:54.374Z+ chown build:build /home/build /work
442026-06-22T23:25:56.373Z+ chmod 0700 /home/build /work
452026-06-22T23:25:56.376Zprocess exited: duration 4421 ms, exit code 0
 
462026-06-22T23:25:56.382Zstarting task 1: "authentication"
472026-06-22T23:25:56.399Zprocess exited: duration 16 ms, exit code 0
 
482026-06-22T23:25:56.406Zstarting task 2: "build"
492026-06-22T23:25:56.409Z+ banner cores
502026-06-22T23:25:56.411Z
512026-06-22T23:25:56.412Z #### #### ##### ###### ####
522026-06-22T23:25:56.412Z # # # # # # # #
532026-06-22T23:25:56.412Z # # # # # ##### ####
542026-06-22T23:25:56.412Z # # # ##### # #
552026-06-22T23:25:56.412Z # # # # # # # # #
562026-06-22T23:25:56.412Z #### #### # # ###### ####
572026-06-22T23:25:56.412Z
582026-06-22T23:25:56.412Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
592026-06-22T23:25:56.417Z+ echo 'input bins dir contains:'
602026-06-22T23:25:56.420Zinput bins dir contains:
612026-06-22T23:25:56.420Z+ ls -ltr /input/build/work/bins
622026-06-22T23:25:56.420Ztotal 1064117
632026-06-22T23:25:56.420Z-rw-r--r-- 1 root root 159846073 Jun 22 23:25 crucible-downstairs.gz
642026-06-22T23:25:56.420Z-rw-r--r-- 1 root root 152702414 Jun 22 23:25 crucible-hammer.gz
652026-06-22T23:25:56.420Z-rw-r--r-- 1 root root 158767498 Jun 22 23:25 crutest.gz
662026-06-22T23:25:56.420Z-rw-r--r-- 1 root root 73155833 Jun 22 23:25 dsc.gz
672026-06-22T23:25:56.420Z+ banner unpack
682026-06-22T23:25:56.420Z
692026-06-22T23:25:56.420Z # # # # ##### ## #### # #
702026-06-22T23:25:56.420Z # # ## # # # # # # # # #
712026-06-22T23:25:56.420Z # # # # # # # # # # ####
722026-06-22T23:25:56.420Z # # # # # ##### ###### # # #
732026-06-22T23:25:56.420Z # # # ## # # # # # # #
742026-06-22T23:25:56.420Z #### # # # # # #### # #
752026-06-22T23:25:56.420Z
762026-06-22T23:25:56.420Z+ mkdir -p /var/tmp/bins
772026-06-22T23:25:56.423Z+ for t in "$input/bins/"*.gz
782026-06-22T23:25:56.423Z++ basename /input/build/work/bins/crucible-downstairs.gz
792026-06-22T23:25:56.423Z+ b=crucible-downstairs.gz
802026-06-22T23:25:56.423Z+ b=crucible-downstairs
812026-06-22T23:25:56.423Z+ gunzip
822026-06-22T23:25:59.104Z+ chmod +x /var/tmp/bins/crucible-downstairs
832026-06-22T23:25:59.107Z+ for t in "$input/bins/"*.gz
842026-06-22T23:25:59.107Z++ basename /input/build/work/bins/crucible-hammer.gz
852026-06-22T23:25:59.107Z+ b=crucible-hammer.gz
862026-06-22T23:25:59.107Z+ b=crucible-hammer
872026-06-22T23:25:59.107Z+ gunzip
882026-06-22T23:26:01.627Z+ chmod +x /var/tmp/bins/crucible-hammer
892026-06-22T23:26:01.631Z+ for t in "$input/bins/"*.gz
902026-06-22T23:26:01.631Z++ basename /input/build/work/bins/crutest.gz
912026-06-22T23:26:01.631Z+ b=crutest.gz
922026-06-22T23:26:01.631Z+ b=crutest
932026-06-22T23:26:01.631Z+ gunzip
942026-06-22T23:26:04.274Z+ chmod +x /var/tmp/bins/crutest
952026-06-22T23:26:04.277Z+ for t in "$input/bins/"*.gz
962026-06-22T23:26:04.277Z++ basename /input/build/work/bins/dsc.gz
972026-06-22T23:26:04.277Z+ b=dsc.gz
982026-06-22T23:26:04.277Z+ b=dsc
992026-06-22T23:26:04.277Z+ gunzip
1002026-06-22T23:26:05.487Z+ chmod +x /var/tmp/bins/dsc
1012026-06-22T23:26:05.490Z+ export BINDIR=/var/tmp/bins
1022026-06-22T23:26:05.491Z+ BINDIR=/var/tmp/bins
1032026-06-22T23:26:05.491Z+ export RUST_BACKTRACE=1
1042026-06-22T23:26:05.491Z+ RUST_BACKTRACE=1
1052026-06-22T23:26:05.491Z+ jobpid=1088
1062026-06-22T23:26:05.491Z+ echo 'Setup debug logging'
1072026-06-22T23:26:05.491Z+ mkdir /tmp/debug
1082026-06-22T23:26:05.491ZSetup debug logging
1092026-06-22T23:26:05.491Z+ sleep 7200
1102026-06-22T23:26:05.494Z+ psrinfo -v
1112026-06-22T23:26:05.494Z+ df -h
1122026-06-22T23:26:05.499Z+ prstat -d d -mLc 1
1132026-06-22T23:26:05.502Z+ iostat -T d -xn 1
1142026-06-22T23:26:05.502Z+ mpstat -T d 1
1152026-06-22T23:26:05.502Z+ vmstat -T d -p 1
1162026-06-22T23:26:05.502Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1172026-06-22T23:26:05.502Z+ banner test_up_unencrypted
1182026-06-22T23:26:05.502Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1192026-06-22T23:26:05.505Z
1202026-06-22T23:26:05.505Z ##### ###### #### ##### # # ##### # # # #
1212026-06-22T23:26:05.505Z # # # # # # # # # # ## #
1222026-06-22T23:26:05.505Z # ##### #### # # # # # # # # # #
1232026-06-22T23:26:05.505Z # # # # # # ##### # # # # #
1242026-06-22T23:26:05.505Z # # # # # # # # # # # ##
1252026-06-22T23:26:05.505Z # ###### #### # ####### #### # ####### #### # #
1262026-06-22T23:26:05.505Z
1272026-06-22T23:26:05.505Z+ ptime -m bash /input/build/work/scripts/test_up.sh -N unencrypted
1282026-06-22T23:26:05.511Z/input/build/work
1292026-06-22T23:26:05.519ZTurn off color for downstairs dump
1302026-06-22T23:26:05.522Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1312026-06-22T23:26:05.522ZCreating 3 downstairs regions
1322026-06-22T23:26:05.753ZStarting 3 downstairs
1332026-06-22T23:26:05.756Zdsc started at PID: 1133
1342026-06-22T23:26:10.804ZDisable automatic restart on all downstairs
1352026-06-22T23:26:10.817Z
1362026-06-22T23:26:10.817ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1372026-06-22T23:26:10.821ZRunning test: span
1382026-06-22T23:26:14.884ZCompleted test: span
1392026-06-22T23:26:19.890ZRunning test: big
1402026-06-22T23:26:23.948ZCompleted test: big
1412026-06-22T23:26:28.955ZRunning test: dep
1422026-06-22T23:26:36.543ZCompleted test: dep
1432026-06-22T23:26:41.549ZRunning test: balloon
1442026-06-22T23:26:48.057ZCompleted test: balloon
1452026-06-22T23:26:53.063ZRunning test: deactivate
1462026-06-22T23:26:57.146ZCompleted test: deactivate
1472026-06-22T23:27:02.153ZRunning hammer
1482026-06-22T23:27:03.370ZRun repair tests
1492026-06-22T23:27:03.374Z/var/tmp/bins/crutest fill -g 31 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1502026-06-22T23:27:03.391Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.884708557Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"task":"crutest"}
1512026-06-22T23:27:03.396Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.884966829Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"task":"crutest"}
1522026-06-22T23:27:03.396Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.88556313Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"task":"crutest"}
1532026-06-22T23:27:03.396Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.885799795Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170}
1542026-06-22T23:27:03.396Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ee79fb76308d9ff9cb3cb44d858aa7578c1a20fe\",\n git_commit_timestamp: \"2026-06-22T23:10:29.000000000Z\",\n git_branch: \"alan/lock-the-pantry\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.885845308Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170}
1552026-06-22T23:27:03.396Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.885855657Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170}
1562026-06-22T23:27:03.396Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.885868315Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170}
1572026-06-22T23:27:03.396Z{"msg":"Crucible 0791674f-1a8f-4269-8c85-817c0d43ceda has session id: f604e72d-39d1-4b1d-9e1c-f8c45f531198","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.886152412Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1582026-06-22T23:27:03.396Z{"msg":"Upstairs opts: Upstairs UUID: 0791674f-1a8f-4269-8c85-817c0d43ceda, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.88616929Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1592026-06-22T23:27:03.396Z{"msg":"Crucible stats registered with UUID: 0791674f-1a8f-4269-8c85-817c0d43ceda","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.886180368Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1602026-06-22T23:27:03.397Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-06-22T23:27:01.886190397Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1612026-06-22T23:27:03.397Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.886359792Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"":"io task","client":"0","":"downstairs","session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1622026-06-22T23:27:03.397Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.886397236Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"":"io task","client":"1","":"downstairs","session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1632026-06-22T23:27:03.397Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.886409134Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"":"io task","client":"2","":"downstairs","session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1642026-06-22T23:27:03.397Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.886535605Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"task":"crutest"}
1652026-06-22T23:27:03.397Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.886549813Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170}
1662026-06-22T23:27:03.397Z{"msg":"0791674f-1a8f-4269-8c85-817c0d43ceda active request set","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.886584618Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1672026-06-22T23:27:03.397Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.886594847Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"client":"0","":"downstairs","session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1682026-06-22T23:27:03.397Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.886605525Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"client":"1","":"downstairs","session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1692026-06-22T23:27:03.397Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.886616243Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"client":"2","":"downstairs","session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1702026-06-22T23:27:03.397Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.886626962Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"":"io task","client":"2","":"downstairs","session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1712026-06-22T23:27:03.397Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.88663804Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"":"io task","client":"0","":"downstairs","session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1722026-06-22T23:27:03.397Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.886663316Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"":"io task","client":"1","":"downstairs","session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1732026-06-22T23:27:03.397Z{"msg":"ds_connection connected from Ok(127.0.0.1:46458)","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.886710669Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"":"io task","client":"0","":"downstairs","session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1742026-06-22T23:27:03.397Z{"msg":"ds_connection connected from Ok(127.0.0.1:44732)","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.886722368Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"":"io task","client":"1","":"downstairs","session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1752026-06-22T23:27:03.397Z{"msg":"ds_connection connected from Ok(127.0.0.1:41260)","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.886733226Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"":"io task","client":"2","":"downstairs","session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1762026-06-22T23:27:03.397Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.887347764Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"client":"1","":"downstairs","session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1772026-06-22T23:27:03.397Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.88737554Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"client":"2","":"downstairs","session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1782026-06-22T23:27:03.397Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.887423343Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"client":"0","":"downstairs","session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1792026-06-22T23:27:03.397Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.887518719Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1802026-06-22T23:27:03.397Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.887536646Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1812026-06-22T23:27:03.397Z{"msg":"[0]R flush_numbers: [1498, 1498, 1498, 1498, 1498]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.887615595Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"":"downstairs","":"downstairs","session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1822026-06-22T23:27:03.397Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.887626773Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"":"downstairs","":"downstairs","session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1832026-06-22T23:27:03.397Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.887637371Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"":"downstairs","":"downstairs","session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1842026-06-22T23:27:03.397Z{"msg":"[1]R flush_numbers: [1498, 1498, 1498, 1498, 1498]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.8876481Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"":"downstairs","":"downstairs","session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1852026-06-22T23:27:03.397Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.887658848Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"":"downstairs","":"downstairs","session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1862026-06-22T23:27:03.397Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.887669337Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"":"downstairs","":"downstairs","session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1872026-06-22T23:27:03.398Z{"msg":"[2]R flush_numbers: [1498, 1498, 1498, 1498, 1498]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.887679965Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"":"downstairs","":"downstairs","session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1882026-06-22T23:27:03.398Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.887690553Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"":"downstairs","":"downstairs","session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1892026-06-22T23:27:03.398Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.887701242Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"":"downstairs","":"downstairs","session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1902026-06-22T23:27:03.398Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.88771142Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"":"downstairs","":"downstairs","session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1912026-06-22T23:27:03.398Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.887722129Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"":"downstairs","":"downstairs","session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1922026-06-22T23:27:03.398Z{"msg":"Next flush: 1499","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.887732197Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"":"downstairs","":"downstairs","session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1932026-06-22T23:27:03.398Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.887742856Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"":"downstairs","":"downstairs","session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1942026-06-22T23:27:03.398Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.887753474Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1952026-06-22T23:27:03.398Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.887763203Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"client":"0","":"downstairs","session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1962026-06-22T23:27:03.398Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.887773491Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"client":"1","":"downstairs","session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1972026-06-22T23:27:03.398Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.88778319Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"client":"2","":"downstairs","session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1982026-06-22T23:27:03.398Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.887793648Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
1992026-06-22T23:27:03.398Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.887803497Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
2002026-06-22T23:27:03.398Z{"msg":"0791674f-1a8f-4269-8c85-817c0d43ceda is now active with session: f604e72d-39d1-4b1d-9e1c-f8c45f531198","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.887814525Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
2012026-06-22T23:27:03.398Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.887824024Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"session_id":"f604e72d-39d1-4b1d-9e1c-f8c45f531198"}
2022026-06-22T23:27:03.398Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.887834132Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170}
2032026-06-22T23:27:03.398Z{"msg":"Activated sub_volume 0791674f-1a8f-4269-8c85-817c0d43ceda","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.887842951Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170}
2042026-06-22T23:27:03.398ZDisk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2052026-06-22T23:27:03.398Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.887856719Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"task":"crutest"}
2062026-06-22T23:27:03.398Z{"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.887889624Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"task":"crutest"}
2072026-06-22T23:27:03.402ZRead and Verify all blocks (0..50 range:false)
2082026-06-22T23:27:03.402Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.894004014Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170----------------------------------------------------------------
2092026-06-22T23:27:03.402Z,"task":"crutest"}
2102026-06-22T23:27:03.402Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2112026-06-22T23:27:03.402Z{States:"msg":" Active ActiveCLIENT: Tests done. All submitted work has been ACK'd" Active,"
2122026-06-22T23:27:03.402Zv"JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2132026-06-22T23:27:03.402Z:0,"name":"crucible","level":30 1002 Acked Read 50 Done," Done Donetime" false:
2142026-06-22T23:27:03.402Z"2026-06-22T23:27:01.894033009Z","hostname":" STATES DS:0 DS:1 DS:2 TOTAL
2152026-06-22T23:27:03.402Zw-01KVRTD4RBMKHF0QR72SRTZ0MZ" Sent ," 0 pid 0 ": 0 1170 0
2162026-06-22T23:27:03.402Z," Done task 1 ":" 1 1 crutest" 3
2172026-06-22T23:27:03.402Z}
2182026-06-22T23:27:03.402Z Skipped 0 0 0 0
2192026-06-22T23:27:03.402Z Error 0 0 0 0
2202026-06-22T23:27:03.402ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2212026-06-22T23:27:03.403ZDownstairs last five completed: 1001 1000
2222026-06-22T23:27:03.403ZUpstairs last five completed: 1002 1001 1000
2232026-06-22T23:27:03.403Z{"msg":"CLIENT: Up:0 ds:1 act:3","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:01.894079892Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"task":"crutest"}
2242026-06-22T23:27:07.402Z----------------------------------------------------------------
2252026-06-22T23:27:07.402Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2262026-06-22T23:27:07.402ZDownstairs last five completed: 1003 1002 1001 1000
2272026-06-22T23:27:07.402ZUpstairs last five completed: 1003 1002 1001 1000
2282026-06-22T23:27:07.405Z{"msg":"CLIENT: Up:0 ds:0 act:3","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.895322454Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"task":"crutest"}
2292026-06-22T23:27:07.405Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.895358508Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170,"task":"crutest"}
2302026-06-22T23:27:07.406Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-06-22T23:27:05.895369237Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1170}
2312026-06-22T23:27:07.409ZRepair setup passed
2322026-06-22T23:27:07.412ZCopy the region for /var/tmp/test_up-build/8830
2332026-06-22T23:27:07.412Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2342026-06-22T23:27:07.415Z/var/tmp/bins/crutest fill -g 32 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2352026-06-22T23:27:07.433Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.925726678Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"task":"crutest"}
2362026-06-22T23:27:07.437Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.925988549Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"task":"crutest"}
2372026-06-22T23:27:07.437Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.92678995Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"task":"crutest"}
2382026-06-22T23:27:07.437Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.927037793Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176}
2392026-06-22T23:27:07.437Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ee79fb76308d9ff9cb3cb44d858aa7578c1a20fe\",\n git_commit_timestamp: \"2026-06-22T23:10:29.000000000Z\",\n git_branch: \"alan/lock-the-pantry\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.927078947Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176}
2402026-06-22T23:27:07.437Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.927092275Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176}
2412026-06-22T23:27:07.437Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.927104843Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176}
2422026-06-22T23:27:07.437Z{"msg":"Crucible bc87ee40-c5ae-4a65-9ff5-7967eb80684b has session id: 16d1ff50-fc32-41ec-abab-ef4cc10c2dc6","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.927440963Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2432026-06-22T23:27:07.437Z{"msg":"Upstairs opts: Upstairs UUID: bc87ee40-c5ae-4a65-9ff5-7967eb80684b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.92746542Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2442026-06-22T23:27:07.437Z{"msg":"Crucible stats registered with UUID: bc87ee40-c5ae-4a65-9ff5-7967eb80684b","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.927476018Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2452026-06-22T23:27:07.437Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-06-22T23:27:05.927490236Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2462026-06-22T23:27:07.437Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.927634205Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"":"io task","client":"0","":"downstairs","session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2472026-06-22T23:27:07.437Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.927661151Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"":"io task","client":"1","":"downstairs","session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2482026-06-22T23:27:07.437Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.927671689Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"":"io task","client":"2","":"downstairs","session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2492026-06-22T23:27:07.437Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.927847583Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"task":"crutest"}
2502026-06-22T23:27:07.437Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.927861301Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176}
2512026-06-22T23:27:07.437Z{"msg":"bc87ee40-c5ae-4a65-9ff5-7967eb80684b active request set","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.927939079Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2522026-06-22T23:27:07.437Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.927953187Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"client":"0","":"downstairs","session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2532026-06-22T23:27:07.437Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.927963616Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"client":"1","":"downstairs","session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2542026-06-22T23:27:07.437Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.927974274Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"client":"2","":"downstairs","session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2552026-06-22T23:27:07.437Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.927988542Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"":"io task","client":"2","":"downstairs","session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2562026-06-22T23:27:07.437Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.928008629Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"":"io task","client":"0","":"downstairs","session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2572026-06-22T23:27:07.437Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.928019627Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"":"io task","client":"1","":"downstairs","session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2582026-06-22T23:27:07.437Z{"msg":"ds_connection connected from Ok(127.0.0.1:57445)","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.928045643Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"":"io task","client":"2","":"downstairs","session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2592026-06-22T23:27:07.437Z{"msg":"ds_connection connected from Ok(127.0.0.1:39586)","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.928061291Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"":"io task","client":"0","":"downstairs","session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2602026-06-22T23:27:07.437Z{"msg":"ds_connection connected from Ok(127.0.0.1:62150)","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.928072449Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"":"io task","client":"1","":"downstairs","session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2612026-06-22T23:27:07.437Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.928716613Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"client":"1","":"downstairs","session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2622026-06-22T23:27:07.437Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.928749868Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"client":"2","":"downstairs","session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2632026-06-22T23:27:07.437Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.928780084Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"client":"0","":"downstairs","session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2642026-06-22T23:27:07.437Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.928954798Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2652026-06-22T23:27:07.437Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.928983234Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2662026-06-22T23:27:07.437Z{"msg":"[0]R flush_numbers: [1499, 1499, 1499, 1499, 1499]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.928994232Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"":"downstairs","":"downstairs","session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2672026-06-22T23:27:07.437Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.929021588Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"":"downstairs","":"downstairs","session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2682026-06-22T23:27:07.437Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.929032416Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"":"downstairs","":"downstairs","session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2692026-06-22T23:27:07.438Z{"msg":"[1]R flush_numbers: [1499, 1499, 1499, 1499, 1499]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.929047414Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"":"downstairs","":"downstairs","session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2702026-06-22T23:27:07.438Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.929057773Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"":"downstairs","":"downstairs","session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2712026-06-22T23:27:07.441Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.929068201Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"":"downstairs","":"downstairs","session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2722026-06-22T23:27:07.441Z{"msg":"[2]R flush_numbers: [1499, 1499, 1499, 1499, 1499]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.929083199Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"":"downstairs","":"downstairs","session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2732026-06-22T23:27:07.441Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.929093527Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"":"downstairs","":"downstairs","session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2742026-06-22T23:27:07.441Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.929103966Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"":"downstairs","":"downstairs","session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2752026-06-22T23:27:07.441Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.929117984Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"":"downstairs","":"downstairs","session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2762026-06-22T23:27:07.441Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.929128132Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"":"downstairs","":"downstairs","session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2772026-06-22T23:27:07.441Z{"msg":"Next flush: 1500","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.92914214Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"":"downstairs","":"downstairs","session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2782026-06-22T23:27:07.441Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.929152199Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"":"downstairs","":"downstairs","session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2792026-06-22T23:27:07.441Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.929166726Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2802026-06-22T23:27:07.441Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.929175905Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"client":"0","":"downstairs","session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2812026-06-22T23:27:07.441Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.929193722Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"client":"1","":"downstairs","session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2822026-06-22T23:27:07.441Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.929203571Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"client":"2","":"downstairs","session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2832026-06-22T23:27:07.441Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.929213729Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2842026-06-22T23:27:07.441Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.929227797Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2852026-06-22T23:27:07.441Z{"msg":"bc87ee40-c5ae-4a65-9ff5-7967eb80684b is now active with session: 16d1ff50-fc32-41ec-abab-ef4cc10c2dc6","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.929238666Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2862026-06-22T23:27:07.441Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.929251874Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"session_id":"16d1ff50-fc32-41ec-abab-ef4cc10c2dc6"}
2872026-06-22T23:27:07.442Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.929261522Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176}
2882026-06-22T23:27:07.442Z{"msg":"Activated sub_volume bc87ee40-c5ae-4a65-9ff5-7967eb80684b","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.92927425Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176}
2892026-06-22T23:27:07.442Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.929282719Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"task":"crutest"}
2902026-06-22T23:27:07.442ZDisk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2912026-06-22T23:27:07.442ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
2922026-06-22T23:27:07.442Z{"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.929649565Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"task":"crutest"}
2932026-06-22T23:27:07.442ZRead and Verify all blocks (0..50 range:false)
2942026-06-22T23:27:07.445Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.935432774Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"task":"crutest"}
2952026-06-22T23:27:07.445Z{"msg":"CLIENT: Tests done. All submitted work has been ACK'd","v":0,"name":"crucible","----------------------------------------------------------------
2962026-06-22T23:27:07.445Zlevel":30 Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
2972026-06-22T23:27:07.445Z,States:"time" Active:" Active Active
2982026-06-22T23:27:07.445Z2026-06-22T23:27:05.93546086Z"JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2992026-06-22T23:27:07.445Z,"hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"task":"crutest" 1002 Acked Read 50}
3002026-06-22T23:27:07.445Z Done Done Done false
3012026-06-22T23:27:07.445Z STATES DS:0 DS:1 DS:2 TOTAL
3022026-06-22T23:27:07.445Z Sent 0 0 0 0
3032026-06-22T23:27:07.445Z Done 1 1 1 3
3042026-06-22T23:27:07.445Z Skipped 0 0 0 0
3052026-06-22T23:27:07.445Z Error 0 0 0 0
3062026-06-22T23:27:07.445ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
3072026-06-22T23:27:07.446ZDownstairs last five completed: 1001 1000
3082026-06-22T23:27:07.446ZUpstairs last five completed: 1002 1001 1000
3092026-06-22T23:27:07.446Z{"msg":"CLIENT: Up:0 ds:1 act:3","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:05.9355279Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"task":"crutest"}
3102026-06-22T23:27:11.444Z----------------------------------------------------------------
3112026-06-22T23:27:11.444Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
3122026-06-22T23:27:11.444ZDownstairs last five completed: 1003 1002 1001 1000
3132026-06-22T23:27:11.444ZUpstairs last five completed: 1003 1002 1001 1000
3142026-06-22T23:27:11.447Z{"msg":"CLIENT: Up:0 ds:0 act:3","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:09.935811184Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"task":"crutest"}
3152026-06-22T23:27:11.447Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:09.935865805Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176,"task":"crutest"}
3162026-06-22T23:27:11.447Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-06-22T23:27:09.935877944Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1176}
3172026-06-22T23:27:11.451ZRepair part 1 passed
3182026-06-22T23:27:11.451Z
3192026-06-22T23:27:11.451ZKill the current downstairs
3202026-06-22T23:27:11.491ZDownstairs 2 stopped
3212026-06-22T23:27:11.491Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
3222026-06-22T23:27:11.495ZNow put back the original so we have a mismatch
3232026-06-22T23:27:11.495Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
3242026-06-22T23:27:11.496ZRestart downstairs with old directory
3252026-06-22T23:27:11.507Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
3262026-06-22T23:27:11.520ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3272026-06-22T23:27:11.523Z 0 000-009 32 32 31 1500 1500 1499 F F F <---
3282026-06-22T23:27:11.523Z 1 010-019 32 32 31 1500 1500 1499 F F F <---
3292026-06-22T23:27:11.523Z 2 020-029 32 32 31 1500 1500 1499 F F F <---
3302026-06-22T23:27:11.523Z 3 030-039 32 32 31 1500 1500 1499 F F F <---
3312026-06-22T23:27:11.523Z 4 040-049 32 32 31 1500 1500 1499 F F F <---
3322026-06-22T23:27:11.523ZMax gen: 32, Max flush: 1500
3332026-06-22T23:27:11.740ZError: Difference in extent metadata found!
3342026-06-22T23:27:11.740Z
3352026-06-22T23:27:11.740ZStack backtrace:
3362026-06-22T23:27:11.740Z 0: anyhow::error::<impl anyhow::Error>::msg
3372026-06-22T23:27:11.740Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14
3382026-06-22T23:27:11.740Z 1: anyhow::__private::format_err
3392026-06-22T23:27:11.740Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13
3402026-06-22T23:27:11.759Zdump test found error as expected
3412026-06-22T23:27:11.762Z
3422026-06-22T23:27:11.762Z
3432026-06-22T23:27:11.762Z/var/tmp/bins/crutest verify --range -g 33 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
3442026-06-22T23:27:11.778Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:10.27063232Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"task":"crutest"}
3452026-06-22T23:27:11.782Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:10.270944344Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"task":"crutest"}
3462026-06-22T23:27:11.782Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:10.271627582Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"task":"crutest"}
3472026-06-22T23:27:11.782Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:10.271882154Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184}
3482026-06-22T23:27:11.782Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ee79fb76308d9ff9cb3cb44d858aa7578c1a20fe\",\n git_commit_timestamp: \"2026-06-22T23:10:29.000000000Z\",\n git_branch: \"alan/lock-the-pantry\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:10.271930927Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184}
3492026-06-22T23:27:11.782Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:10.271944735Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184}
3502026-06-22T23:27:11.782Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:10.271957273Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184}
3512026-06-22T23:27:11.782Z{"msg":"Crucible b9277c02-151c-43be-8af9-881adcc61cef has session id: c066b634-7753-4abd-9e74-840fab6cd70f","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:10.272324808Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3522026-06-22T23:27:11.782Z{"msg":"Upstairs opts: Upstairs UUID: b9277c02-151c-43be-8af9-881adcc61cef, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:10.272359333Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3532026-06-22T23:27:11.782Z{"msg":"Crucible stats registered with UUID: b9277c02-151c-43be-8af9-881adcc61cef","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:10.272375501Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3542026-06-22T23:27:11.782Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-06-22T23:27:10.272386759Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3552026-06-22T23:27:11.782Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:10.272520329Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"io task","client":"1","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3562026-06-22T23:27:11.782Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:10.272544576Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"io task","client":"2","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3572026-06-22T23:27:11.782Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:10.272562833Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"io task","client":"0","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3582026-06-22T23:27:11.782Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:10.272750325Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"task":"crutest"}
3592026-06-22T23:27:11.782Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:10.272790749Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184}
3602026-06-22T23:27:11.783Z{"msg":"b9277c02-151c-43be-8af9-881adcc61cef active request set","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:10.272811816Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3612026-06-22T23:27:11.783Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:10.272826604Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"client":"0","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3622026-06-22T23:27:11.783Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:10.272837162Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"client":"1","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3632026-06-22T23:27:11.783Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:10.27285215Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"client":"2","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3642026-06-22T23:27:11.783Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:10.272862598Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"io task","client":"0","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3652026-06-22T23:27:11.783Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:10.272877556Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"io task","client":"2","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3662026-06-22T23:27:11.783Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:10.272887535Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"io task","client":"1","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3672026-06-22T23:27:11.783Z{"msg":"ds_connection connected from Ok(127.0.0.1:52590)","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:10.272936907Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"io task","client":"1","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3682026-06-22T23:27:11.783Z{"msg":"ds_connection connected from Ok(127.0.0.1:43862)","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:10.272948855Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"io task","client":"0","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3692026-06-22T23:27:11.783Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-06-22T23:27:10.272961843Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"io task","client":"2","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3702026-06-22T23:27:11.783Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-06-22T23:27:10.272974782Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"io task","client":"2","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3712026-06-22T23:27:11.783Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-06-22T23:27:10.272995878Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3722026-06-22T23:27:11.783Z{"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:10.27305573Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"io task","client":"2","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3732026-06-22T23:27:11.783Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:10.273070847Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"io task","client":"2","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3742026-06-22T23:27:11.783Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:10.273663709Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"client":"0","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3752026-06-22T23:27:11.783Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:10.273685986Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"client":"1","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3762026-06-22T23:27:11.783Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:10.273836673Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3772026-06-22T23:27:11.783Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:10.273850661Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3782026-06-22T23:27:21.783Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.274296835Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"io task","client":"2","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3792026-06-22T23:27:21.784Z{"msg":"ds_connection connected from Ok(127.0.0.1:44419)","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.274354416Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"io task","client":"2","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3802026-06-22T23:27:21.789Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.275109184Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"client":"2","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3812026-06-22T23:27:21.789Z{"msg":"[0]R flush_numbers: [1500, 1500, 1500, 1500, 1500]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.275311674Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"downstairs","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3822026-06-22T23:27:21.789Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.275327791Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"downstairs","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3832026-06-22T23:27:21.789Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.27533909Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"downstairs","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3842026-06-22T23:27:21.789Z{"msg":"[1]R flush_numbers: [1500, 1500, 1500, 1500, 1500]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.275349968Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"downstairs","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3852026-06-22T23:27:21.789Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.275364636Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"downstairs","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3862026-06-22T23:27:21.789Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.275375754Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"downstairs","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3872026-06-22T23:27:21.789Z{"msg":"[2]R flush_numbers: [1499, 1499, 1499, 1499, 1499]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.275390062Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"downstairs","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3882026-06-22T23:27:21.789Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.27540426Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"downstairs","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3892026-06-22T23:27:21.789Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.275414778Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"downstairs","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3902026-06-22T23:27:21.789Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.275428676Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"downstairs","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3912026-06-22T23:27:21.789Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.275439445Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"downstairs","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3922026-06-22T23:27:21.789Z{"msg":"Next flush: 1501","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.275453153Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"downstairs","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3932026-06-22T23:27:21.789Z{"msg":"extent 0 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1499, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.27546943Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"mend","":"downstairs","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3942026-06-22T23:27:21.789Z{"msg":"extent:0 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1499, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.275488527Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"mend","":"downstairs","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3952026-06-22T23:27:21.789Z{"msg":"found dest for source 0 for extent at index 0 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.275503825Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"mend","":"downstairs","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3962026-06-22T23:27:21.789Z{"msg":"extent 1 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1499, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.275518273Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"mend","":"downstairs","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3972026-06-22T23:27:21.789Z{"msg":"extent:1 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1499, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.27553702Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"mend","":"downstairs","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3982026-06-22T23:27:21.789Z{"msg":"found dest for source 0 for extent at index 1 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.275547959Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"mend","":"downstairs","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
3992026-06-22T23:27:21.789Z{"msg":"extent 2 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1499, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.275566586Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"mend","":"downstairs","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
4002026-06-22T23:27:21.789Z{"msg":"extent:2 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1499, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.275585433Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"mend","":"downstairs","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
4012026-06-22T23:27:21.789Z{"msg":"found dest for source 0 for extent at index 2 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.275600511Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"mend","":"downstairs","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
4022026-06-22T23:27:21.790Z{"msg":"extent 3 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1499, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.275618788Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"mend","":"downstairs","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
4032026-06-22T23:27:21.790Z{"msg":"extent:3 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1499, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.275637155Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"mend","":"downstairs","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
4042026-06-22T23:27:21.790Z{"msg":"found dest for source 0 for extent at index 3 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.275651843Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"mend","":"downstairs","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
4052026-06-22T23:27:21.790Z{"msg":"extent 4 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1499, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.275666771Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"mend","":"downstairs","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
4062026-06-22T23:27:21.790Z{"msg":"extent:4 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1499, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.275685238Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"mend","":"downstairs","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
4072026-06-22T23:27:21.790Z{"msg":"found dest for source 0 for extent at index 4 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.275696506Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"mend","":"downstairs","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
4082026-06-22T23:27:21.790Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.275717953Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"client":"0","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
4092026-06-22T23:27:21.790Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.275732971Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"client":"1","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
4102026-06-22T23:27:21.790Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.275744429Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"client":"2","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
4112026-06-22T23:27:21.790Z{"msg":"Full repair list: {ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.275760667Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"downstairs","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
4122026-06-22T23:27:21.790Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 1501, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 1501, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 1501, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 1501, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 1501, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.275861952Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"downstairs","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
4132026-06-22T23:27:21.790Z{"msg":"starting reconciliation 2ffdde40-8916-44e6-8ed9-f19586c85e10: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.27587381Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"downstairs","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
4142026-06-22T23:27:21.790Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.275883749Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"client":"2","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
4152026-06-22T23:27:21.796Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.287057846Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"client":"2","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
4162026-06-22T23:27:21.799Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.28904354Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"client":"2","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
4172026-06-22T23:27:21.802Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.290887096Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"client":"2","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
4182026-06-22T23:27:21.802Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.29287246Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"client":"2","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
4192026-06-22T23:27:21.805Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.294709557Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"":"downstairs","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
4202026-06-22T23:27:21.805Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.294736983Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"client":"0","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
4212026-06-22T23:27:21.805Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.294747941Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"client":"1","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
4222026-06-22T23:27:21.805Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.294758059Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"client":"2","":"downstairs","session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
4232026-06-22T23:27:21.805Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.294772817Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
4242026-06-22T23:27:21.805Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.294782236Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
4252026-06-22T23:27:21.805Z{"msg":"b9277c02-151c-43be-8af9-881adcc61cef is now active with session: c066b634-7753-4abd-9e74-840fab6cd70f","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.294797024Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"session_id":"c066b634-7753-4abd-9e74-840fab6cd70f"}
4262026-06-22T23:27:21.805Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.294806822Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184}
4272026-06-22T23:27:21.805Z{"msg":"Activated sub_volume b9277c02-151c-43be-8af9-881adcc61cef","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.29481956Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184}
4282026-06-22T23:27:21.805Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.294839007Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"task":"crutest"}
4292026-06-22T23:27:21.805ZDisk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
4302026-06-22T23:27:21.805ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4312026-06-22T23:27:21.805ZRead and Verify all blocks (0..50 range:true)
4322026-06-22T23:27:21.808Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.298744116Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"task":"crutest"}
4332026-06-22T23:27:21.808Z{"msg":"Verify test completed","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.298768613Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"task":"crutest"}
4342026-06-22T23:27:21.808Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.298778571Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"task":"crutest"}
4352026-06-22T23:27:21.809Z{"msg":"CLIENT: Tests done. All submitted work has been ACK'd","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.298792049Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"task":"crutest"}
4362026-06-22T23:27:21.809Z----------------------------------------------------------------
4372026-06-22T23:27:21.809Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
4382026-06-22T23:27:21.809ZStates: Active Active Active
4392026-06-22T23:27:21.809ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4402026-06-22T23:27:21.809Z 1000 Acked Read 50 Done Done Done false
4412026-06-22T23:27:21.809Z STATES DS:0 DS:1 DS:2 TOTAL
4422026-06-22T23:27:21.809Z Sent 0 0 0 0
4432026-06-22T23:27:21.809Z Done 1 1 1 3
4442026-06-22T23:27:21.809Z Skipped 0 0 0 0
4452026-06-22T23:27:21.809Z Error 0 0 0 0
4462026-06-22T23:27:21.809ZLast Flush: None None None
4472026-06-22T23:27:21.809ZDownstairs last five completed:
4482026-06-22T23:27:21.809ZUpstairs last five completed: 1000
4492026-06-22T23:27:21.809Z{"msg":"CLIENT: Up:0 ds:1 act:3","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:20.298907992Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"task":"crutest"}
4502026-06-22T23:27:25.809Z----------------------------------------------------------------
4512026-06-22T23:27:25.810Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
4522026-06-22T23:27:25.810ZDownstairs last five completed: 1001 1000
4532026-06-22T23:27:25.810ZUpstairs last five completed: 1001 1000
4542026-06-22T23:27:25.813Z{"msg":"CLIENT: Up:0 ds:0 act:3","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:24.299878793Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"task":"crutest"}
4552026-06-22T23:27:25.813Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-06-22T23:27:24.299905969Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184,"task":"crutest"}
4562026-06-22T23:27:25.813Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-06-22T23:27:24.299916178Z","hostname":"w-01KVRTD4RBMKHF0QR72SRTZ0MZ","pid":1184}
4572026-06-22T23:27:25.816ZRepair part 2 passed
4582026-06-22T23:27:25.816Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4592026-06-22T23:27:25.827ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4602026-06-22T23:27:25.827Z 0 000-009 32 32 32 1500 1500 1500 F F F
4612026-06-22T23:27:25.827Z 1 010-019 32 32 32 1500 1500 1500 F F F
4622026-06-22T23:27:25.827Z 2 020-029 32 32 32 1500 1500 1500 F F F
4632026-06-22T23:27:25.827Z 3 030-039 32 32 32 1500 1500 1500 F F F
4642026-06-22T23:27:25.827Z 4 040-049 32 32 32 1500 1500 1500 F F F
4652026-06-22T23:27:25.830ZMax gen: 32, Max flush: 1500
4662026-06-22T23:27:25.830Zdump test passed
4672026-06-22T23:27:25.830Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
4682026-06-22T23:27:25.840Z Extent 1
4692026-06-22T23:27:25.840ZGEN 32 32 32
4702026-06-22T23:27:25.840ZFLUSH_ID 1500 1500 1500
4712026-06-22T23:27:25.840ZDIRTY
4722026-06-22T23:27:25.840Z
4732026-06-22T23:27:25.841ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
4742026-06-22T23:27:25.844Z010 A A A A A A
4752026-06-22T23:27:25.844Z011 A A A A A A
4762026-06-22T23:27:25.844Z012 A A A A A A
4772026-06-22T23:27:25.844Z013 A A A A A A
4782026-06-22T23:27:25.844Z014 A A A A A A
4792026-06-22T23:27:25.844Z015 A A A A A A
4802026-06-22T23:27:25.847Z016 A A A A A A
4812026-06-22T23:27:25.847Z017 A A A A A A
4822026-06-22T23:27:25.847Z018 A A A A A A
4832026-06-22T23:27:25.847Z019 A A A A A A
4842026-06-22T23:27:25.849Zdump extent test passed
4852026-06-22T23:27:25.850Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
4862026-06-22T23:27:25.858ZExtent 2 Block in extent 0 Actual block 20
4872026-06-22T23:27:25.858Z
4882026-06-22T23:27:25.861Z DATA SHA256 VER
4892026-06-22T23:27:25.861Z------ ---------------------------------------------------------------- ---
4902026-06-22T23:27:25.861Z 0 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
4912026-06-22T23:27:25.861Z 1 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
4922026-06-22T23:27:25.861Z 2 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
4932026-06-22T23:27:25.861Z
4942026-06-22T23:27:25.861ZNONCES 0 1 2 DIFF
4952026-06-22T23:27:25.861Z------ ------------------------ ------------------------ ------------------------ -----
4962026-06-22T23:27:25.861Z
4972026-06-22T23:27:25.861Z TAGS 0 1 2 DIFF
4982026-06-22T23:27:25.861Z------ -------------------------------- -------------------------------- -------------------------------- -----
4992026-06-22T23:27:25.861Z
5002026-06-22T23:27:25.861ZHASHES 0 1 2 DIFF
5012026-06-22T23:27:25.861Z------ ---------------- ---------------- ---------------- -----
5022026-06-22T23:27:25.861Z 0 42916ab6fa364c70 42916ab6fa364c70 42916ab6fa364c70
5032026-06-22T23:27:25.861Z
5042026-06-22T23:27:25.864Zdump block test passed
5052026-06-22T23:27:25.864ZInitial upstairs tests have completed, stopping all downstairs
5062026-06-22T23:27:30.882ZCreating 4 larger downstairs regions
5072026-06-22T23:27:31.001ZStarting 4 downstairs
5082026-06-22T23:27:36.007Zdsc restarted at PID: 1197
5092026-06-22T23:27:36.072ZNow do the replace-reconcile test
5102026-06-22T23:27:36.086ZUsing 8840 for the replacement port
5112026-06-22T23:31:09.984ZCompleted test: replace-reconcile
5122026-06-22T23:31:09.987ZNow do the replace-before-active test
5132026-06-22T23:33:58.181ZCompleted test: replace-before-active
5142026-06-22T23:33:58.181ZAll tests have completed, stopping all downstairs
5152026-06-22T23:33:58.198Z
5162026-06-22T23:33:58.198ZAll Tests have passed
5172026-06-22T23:33:58.201Z7:52 Test duration
5182026-06-22T23:33:58.201Z
5192026-06-22T23:33:58.201Zreal 7:52.620164423
5202026-06-22T23:33:58.201Zuser 41.193230182
5212026-06-22T23:33:58.201Zsys 5.165688675
5222026-06-22T23:33:58.201Ztrap 0.062228544
5232026-06-22T23:33:58.201Ztflt 0.022602765
5242026-06-22T23:33:58.201Zdflt 0.008632835
5252026-06-22T23:33:58.201Zkflt 0.001247002
5262026-06-22T23:33:58.201Zlock 4:06:13.286848067
5272026-06-22T23:33:58.201Zslp 20:41.399803007
5282026-06-22T23:33:58.201Zlat 4.053735285
5292026-06-22T23:33:58.201Zstop 0.010781443
5302026-06-22T23:33:58.201Z+ echo 'test-up-unencrypted ends'
5312026-06-22T23:33:58.201Ztest-up-unencrypted ends
5322026-06-22T23:34:03.203Zprocess exited: duration 481719 ms, exit code 0
5332026-06-22T23:34:03.203Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
5342026-06-22T23:35:03.239Zexec warning: : stdout descriptor may be held open by a background process; giving up!
5352026-06-22T23:35:03.239Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
5362026-06-22T23:35:03.252Zfound 14 output files
5372026-06-22T23:35:03.253Zuploading: /tmp/test_up-build/dsc-out.txt (47508 bytes)
5382026-06-22T23:35:04.266Zuploaded: /tmp/test_up-build/dsc-out.txt
5392026-06-22T23:35:04.266Zuploading: /tmp/test_up-build/test_up_out.txt (22520194 bytes)
5402026-06-22T23:35:05.410Zuploaded: /tmp/test_up-build/test_up_out.txt
5412026-06-22T23:35:05.410Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (2494 bytes)
5422026-06-22T23:35:06.421Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
5432026-06-22T23:35:06.421Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (548330 bytes)
5442026-06-22T23:35:07.439Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
5452026-06-22T23:35:07.442Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (978661 bytes)
5462026-06-22T23:35:08.459Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
5472026-06-22T23:35:08.459Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (543025 bytes)
5482026-06-22T23:35:09.480Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
5492026-06-22T23:35:09.480Zuploading: /tmp/debug/df.txt (1270 bytes)
5502026-06-22T23:35:10.489Zuploaded: /tmp/debug/df.txt
5512026-06-22T23:35:10.489Zuploading: /tmp/debug/dtrace.txt (488990 bytes)
5522026-06-22T23:35:10.503Zupload warning: file "/tmp/debug/dtrace.txt" changed size mid upload: 488990 -> 586187
5532026-06-22T23:35:11.510Zuploaded: /tmp/debug/dtrace.txt
5542026-06-22T23:35:11.510Zuploading: /tmp/debug/iostat.txt (159799 bytes)
5552026-06-22T23:35:11.520Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 159799 -> 162175
5562026-06-22T23:35:12.528Zuploaded: /tmp/debug/iostat.txt
5572026-06-22T23:35:12.528Zuploading: /tmp/debug/mpstat.txt (390774 bytes)
5582026-06-22T23:35:12.540Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 390774 -> 397308
5592026-06-22T23:35:13.547Zuploaded: /tmp/debug/mpstat.txt
5602026-06-22T23:35:13.547Zuploading: /tmp/debug/paging.txt (68788 bytes)
5612026-06-22T23:35:13.555Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 68788 -> 70066
5622026-06-22T23:35:14.562Zuploaded: /tmp/debug/paging.txt
5632026-06-22T23:35:14.562Zuploading: /tmp/debug/prstat.txt (734996 bytes)
5642026-06-22T23:35:14.577Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 734996 -> 749257
5652026-06-22T23:35:15.584Zuploaded: /tmp/debug/prstat.txt
5662026-06-22T23:35:15.584Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
5672026-06-22T23:35:16.593Zuploaded: /tmp/debug/psrinfo.txt
5682026-06-22T23:35:16.593Zuploading: /tmp/debug/upstairs-info.txt (53907 bytes)
5692026-06-22T23:35:17.605Zuploaded: /tmp/debug/upstairs-info.txt