01KY0V34SAGM91EHNBS5S2GGZA: test-up-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01KY0V3QWXKC6MXTCC25XQSE7Z

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12026-07-20T22:54:41.904Zjob dependencies complete; ready to run (waiting for 12 m 16 s)
22026-07-20T22:55:42.204Zjob assigned to worker 01KY0VTFMWMEWPCY0ZV7HPTC2H [factory aws, i-0a83fcbb225530e95] (queued for 1 m 0 s)
32026-07-20T22:55:48.663Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42026-07-20T22:55:50.228Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52026-07-20T22:55:50.231Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62026-07-20T22:55:51.688Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72026-07-20T22:55:51.693Zdownloading input: /input/build/work/bins/crutest.gz
82026-07-20T22:55:53.214Zdownloaded input: /input/build/work/bins/crutest.gz
92026-07-20T22:55:53.221Zdownloading input: /input/build/work/bins/dsc.gz
102026-07-20T22:55:53.915Zdownloaded input: /input/build/work/bins/dsc.gz
112026-07-20T22:55:53.919Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122026-07-20T22:55:53.944Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132026-07-20T22:55:53.948Zdownloading input: /input/build/work/scripts/test_ds.sh
142026-07-20T22:55:53.977Zdownloaded input: /input/build/work/scripts/test_ds.sh
152026-07-20T22:55:53.984Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162026-07-20T22:55:54.013Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172026-07-20T22:55:54.013Zdownloading input: /input/build/work/scripts/test_repair.sh
182026-07-20T22:55:54.037Zdownloaded input: /input/build/work/scripts/test_repair.sh
192026-07-20T22:55:54.037Zdownloading input: /input/build/work/scripts/test_replay.sh
202026-07-20T22:55:54.067Zdownloaded input: /input/build/work/scripts/test_replay.sh
212026-07-20T22:55:54.067Zdownloading input: /input/build/work/scripts/test_up.sh
222026-07-20T22:55:54.102Zdownloaded input: /input/build/work/scripts/test_up.sh
232026-07-20T22:55:54.102Zdownloading input: /input/build/work/scripts/upstairs_info.d
242026-07-20T22:55:54.133Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252026-07-20T22:55:54.133Zdownloading input: /input/build/tmp/cargo-test-out.log
262026-07-20T22:55:54.741Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272026-07-20T22:55:54.741Zstarting task 0: "setup"
282026-07-20T22:55:54.777Z++ uname -s
292026-07-20T22:55:54.777Z+ kern=SunOS
302026-07-20T22:55:54.777Z+ build_user=build
312026-07-20T22:55:54.777Z+ build_uid=12345
322026-07-20T22:55:54.778Z+ work_dir=/work
332026-07-20T22:55:54.778Z+ input_dir=/input
342026-07-20T22:55:54.778Z+ [[ 0 == 12345 ]]
352026-07-20T22:55:54.778Z+ case "$kern" in
362026-07-20T22:55:54.778Z+ groupadd -g 12345 build
372026-07-20T22:55:54.778Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382026-07-20T22:55:56.719Z+ zfs create -o mountpoint=/work rpool/work
392026-07-20T22:55:56.972Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402026-07-20T22:55:56.976Z+ home_fs=zfs
412026-07-20T22:55:56.977Z+ [[ zfs == autofs ]]
422026-07-20T22:55:56.977Z+ mkdir -p /home/build
432026-07-20T22:55:56.977Z+ chown build:build /home/build /work
442026-07-20T22:55:57.975Z+ chmod 0700 /home/build /work
452026-07-20T22:55:57.978Zprocess exited: duration 3274 ms, exit code 0
 
462026-07-20T22:55:57.984Zstarting task 1: "authentication"
472026-07-20T22:55:58.002Zprocess exited: duration 17 ms, exit code 0
 
482026-07-20T22:55:58.008Zstarting task 2: "build"
492026-07-20T22:55:58.012Z+ banner cores
502026-07-20T22:55:58.015Z
512026-07-20T22:55:58.015Z #### #### ##### ###### ####
522026-07-20T22:55:58.016Z # # # # # # # #
532026-07-20T22:55:58.016Z # # # # # ##### ####
542026-07-20T22:55:58.016Z # # # ##### # #
552026-07-20T22:55:58.016Z # # # # # # # # #
562026-07-20T22:55:58.016Z #### #### # # ###### ####
572026-07-20T22:55:58.016Z
582026-07-20T22:55:58.016Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
592026-07-20T22:55:58.022Z+ echo 'input bins dir contains:'
602026-07-20T22:55:58.026Z+ ls -ltr /input/build/work/bins
612026-07-20T22:55:58.026Zinput bins dir contains:
622026-07-20T22:55:58.026Ztotal 1013422
632026-07-20T22:55:58.026Z-rw-r--r-- 1 root root 152704177 Jul 20 22:55 crucible-downstairs.gz
642026-07-20T22:55:58.026Z-rw-r--r-- 1 root root 145147730 Jul 20 22:55 crucible-hammer.gz
652026-07-20T22:55:58.026Z-rw-r--r-- 1 root root 151756077 Jul 20 22:55 crutest.gz
662026-07-20T22:55:58.026Z-rw-r--r-- 1 root root 68924863 Jul 20 22:55 dsc.gz
672026-07-20T22:55:58.026Z+ banner unpack
682026-07-20T22:55:58.026Z
692026-07-20T22:55:58.026Z # # # # ##### ## #### # #
702026-07-20T22:55:58.026Z # # ## # # # # # # # # #
712026-07-20T22:55:58.026Z # # # # # # # # # # ####
722026-07-20T22:55:58.026Z # # # # # ##### ###### # # #
732026-07-20T22:55:58.026Z # # # ## # # # # # # #
742026-07-20T22:55:58.026Z #### # # # # # #### # #
752026-07-20T22:55:58.026Z
762026-07-20T22:55:58.026Z+ mkdir -p /var/tmp/bins
772026-07-20T22:55:58.026Z+ for t in "$input/bins/"*.gz
782026-07-20T22:55:58.026Z++ basename /input/build/work/bins/crucible-downstairs.gz
792026-07-20T22:55:58.026Z+ b=crucible-downstairs.gz
802026-07-20T22:55:58.026Z+ b=crucible-downstairs
812026-07-20T22:55:58.027Z+ gunzip
822026-07-20T22:56:00.620Z+ chmod +x /var/tmp/bins/crucible-downstairs
832026-07-20T22:56:00.623Z+ for t in "$input/bins/"*.gz
842026-07-20T22:56:00.626Z++ basename /input/build/work/bins/crucible-hammer.gz
852026-07-20T22:56:00.626Z+ b=crucible-hammer.gz
862026-07-20T22:56:00.626Z+ b=crucible-hammer
872026-07-20T22:56:00.626Z+ gunzip
882026-07-20T22:56:03.047Z+ chmod +x /var/tmp/bins/crucible-hammer
892026-07-20T22:56:03.050Z+ for t in "$input/bins/"*.gz
902026-07-20T22:56:03.053Z++ basename /input/build/work/bins/crutest.gz
912026-07-20T22:56:03.053Z+ b=crutest.gz
922026-07-20T22:56:03.053Z+ b=crutest
932026-07-20T22:56:03.053Z+ gunzip
942026-07-20T22:56:05.609Z+ chmod +x /var/tmp/bins/crutest
952026-07-20T22:56:05.612Z+ for t in "$input/bins/"*.gz
962026-07-20T22:56:05.612Z++ basename /input/build/work/bins/dsc.gz
972026-07-20T22:56:05.612Z+ b=dsc.gz
982026-07-20T22:56:05.612Z+ b=dsc
992026-07-20T22:56:05.612Z+ gunzip
1002026-07-20T22:56:06.753Z+ chmod +x /var/tmp/bins/dsc
1012026-07-20T22:56:06.756Z+ export BINDIR=/var/tmp/bins
1022026-07-20T22:56:06.756Z+ BINDIR=/var/tmp/bins
1032026-07-20T22:56:06.756Z+ export RUST_BACKTRACE=1
1042026-07-20T22:56:06.756Z+ RUST_BACKTRACE=1
1052026-07-20T22:56:06.756Z+ jobpid=1091
1062026-07-20T22:56:06.756Z+ echo 'Setup debug logging'
1072026-07-20T22:56:06.756Z+ mkdir /tmp/debug
1082026-07-20T22:56:06.756ZSetup debug logging
1092026-07-20T22:56:06.756Z+ sleep 3600
1102026-07-20T22:56:06.760Z+ psrinfo -v
1112026-07-20T22:56:06.760Z+ df -h
1122026-07-20T22:56:06.765Z+ prstat -d d -mLc 1
1132026-07-20T22:56:06.767Z+ iostat -T d -xn 1
1142026-07-20T22:56:06.767Z+ mpstat -T d 1
1152026-07-20T22:56:06.767Z+ vmstat -T d -p 1
1162026-07-20T22:56:06.767Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1172026-07-20T22:56:06.767Z+ banner test_up_encrypted
1182026-07-20T22:56:06.767Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1192026-07-20T22:56:06.770Z
1202026-07-20T22:56:06.770Z ##### ###### #### ##### # # ##### ###### # #
1212026-07-20T22:56:06.770Z # # # # # # # # # ## #
1222026-07-20T22:56:06.770Z # ##### #### # # # # # ##### # # #
1232026-07-20T22:56:06.770Z # # # # # # ##### # # # #
1242026-07-20T22:56:06.770Z # # # # # # # # # # ##
1252026-07-20T22:56:06.770Z # ###### #### # ####### #### # ####### ###### # #
1262026-07-20T22:56:06.770Z
1272026-07-20T22:56:06.770Z+ ptime -m bash /input/build/work/scripts/test_up.sh -N encrypted
1282026-07-20T22:56:06.777Z/input/build/work
1292026-07-20T22:56:06.785ZTurn off color for downstairs dump
1302026-07-20T22:56:06.797ZUpstairs using key: vG6yGA6icbhyoJRLwZ728fUSSYe4yhOxWoMKAwnMIcA=
1312026-07-20T22:56:06.800Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1322026-07-20T22:56:06.800ZCreating 3 downstairs regions
1332026-07-20T22:56:07.019ZStarting 3 downstairs
1342026-07-20T22:56:07.023Zdsc started at PID: 1137
1352026-07-20T22:56:12.072ZDisable automatic restart on all downstairs
1362026-07-20T22:56:12.084Z
1372026-07-20T22:56:12.084ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1382026-07-20T22:56:12.087ZRunning test: span
1392026-07-20T22:56:16.181ZCompleted test: span
1402026-07-20T22:56:21.188ZRunning test: big
1412026-07-20T22:56:25.351ZCompleted test: big
1422026-07-20T22:56:30.358ZRunning test: dep
1432026-07-20T22:57:01.785ZCompleted test: dep
1442026-07-20T22:57:06.791ZRunning test: balloon
1452026-07-20T22:57:32.035ZCompleted test: balloon
1462026-07-20T22:57:37.037ZRunning test: deactivate
1472026-07-20T22:57:41.363ZCompleted test: deactivate
1482026-07-20T22:57:46.370ZRunning hammer
1492026-07-20T22:57:58.648ZRun repair tests
1502026-07-20T22:57:58.651Z/var/tmp/bins/crutest fill -g 31 -q --key vG6yGA6icbhyoJRLwZ728fUSSYe4yhOxWoMKAwnMIcA= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1512026-07-20T22:57:58.670Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.750487803Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"task":"crutest"}
1522026-07-20T22:57:58.674Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.750831834Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"task":"crutest"}
1532026-07-20T22:57:58.674Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.751670902Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"task":"crutest"}
1542026-07-20T22:57:58.674Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.751914481Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174}
1552026-07-20T22:57:58.674Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a81b42ee266cbdd6371f13d7b3f8dd00b79d3cab\",\n git_commit_timestamp: \"2026-07-20T22:41:44.000000000Z\",\n git_branch: \"renovate/crate-rand-vulnerability\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.751943099Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174}
1562026-07-20T22:57:58.674Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.751956418Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174}
1572026-07-20T22:57:58.674Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.751968656Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174}
1582026-07-20T22:57:58.674Z{"msg":"Crucible 456d3519-b434-4cdf-83e3-9a6a413a5647 has session id: ddfb31c5-9662-4c69-bb26-7a85684284da","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.752430997Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1592026-07-20T22:57:58.674Z{"msg":"Upstairs opts: Upstairs UUID: 456d3519-b434-4cdf-83e3-9a6a413a5647, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.752474523Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1602026-07-20T22:57:58.674Z{"msg":"Crucible stats registered with UUID: 456d3519-b434-4cdf-83e3-9a6a413a5647","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.752493481Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1612026-07-20T22:57:58.674Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-20T22:57:57.752504771Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1622026-07-20T22:57:58.674Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.752636919Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"":"io task","client":"1","":"downstairs","session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1632026-07-20T22:57:58.674Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.752676766Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"":"io task","client":"2","":"downstairs","session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1642026-07-20T22:57:58.674Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.752688045Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"":"io task","client":"0","":"downstairs","session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1652026-07-20T22:57:58.674Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.752882028Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"task":"crutest"}
1662026-07-20T22:57:58.674Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.752899697Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174}
1672026-07-20T22:57:58.674Z{"msg":"456d3519-b434-4cdf-83e3-9a6a413a5647 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.752940253Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1682026-07-20T22:57:58.674Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.752968541Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"client":"0","":"downstairs","session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1692026-07-20T22:57:58.674Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.75297982Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"client":"1","":"downstairs","session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1702026-07-20T22:57:58.674Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.752990629Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"client":"2","":"downstairs","session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1712026-07-20T22:57:58.674Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.753000498Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"":"io task","client":"2","":"downstairs","session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1722026-07-20T22:57:58.674Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.753011557Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"":"io task","client":"0","":"downstairs","session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1732026-07-20T22:57:58.674Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.753021836Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"":"io task","client":"1","":"downstairs","session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1742026-07-20T22:57:58.674Z{"msg":"ds_connection connected from Ok(127.0.0.1:49751)","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.753070452Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"":"io task","client":"2","":"downstairs","session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1752026-07-20T22:57:58.674Z{"msg":"ds_connection connected from Ok(127.0.0.1:58852)","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.753086311Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"":"io task","client":"0","":"downstairs","session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1762026-07-20T22:57:58.674Z{"msg":"ds_connection connected from Ok(127.0.0.1:62711)","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.75309709Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"":"io task","client":"1","":"downstairs","session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1772026-07-20T22:57:58.674Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.753806269Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"client":"0","":"downstairs","session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1782026-07-20T22:57:58.674Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.753836946Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"client":"1","":"downstairs","session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1792026-07-20T22:57:58.674Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.753848385Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"client":"2","":"downstairs","session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1802026-07-20T22:57:58.681Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.754020311Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1812026-07-20T22:57:58.681Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.754044239Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1822026-07-20T22:57:58.681Z{"msg":"[0]R flush_numbers: [1498, 1498, 1498, 1498, 1498]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.754055468Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"":"downstairs","":"downstairs","session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1832026-07-20T22:57:58.681Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.754066537Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"":"downstairs","":"downstairs","session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1842026-07-20T22:57:58.681Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.754077376Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"":"downstairs","":"downstairs","session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1852026-07-20T22:57:58.681Z{"msg":"[1]R flush_numbers: [1498, 1498, 1498, 1498, 1498]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.754088395Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"":"downstairs","":"downstairs","session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1862026-07-20T22:57:58.681Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.754099174Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"":"downstairs","":"downstairs","session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1872026-07-20T22:57:58.681Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.754109803Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"":"downstairs","":"downstairs","session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1882026-07-20T22:57:58.682Z{"msg":"[2]R flush_numbers: [1498, 1498, 1498, 1498, 1498]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.754120512Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"":"downstairs","":"downstairs","session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1892026-07-20T22:57:58.682Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.754130221Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"":"downstairs","":"downstairs","session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1902026-07-20T22:57:58.682Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.75414077Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"":"downstairs","":"downstairs","session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1912026-07-20T22:57:58.682Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.754151069Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"":"downstairs","":"downstairs","session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1922026-07-20T22:57:58.682Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.754161798Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"":"downstairs","":"downstairs","session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1932026-07-20T22:57:58.682Z{"msg":"Next flush: 1499","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.754171948Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"":"downstairs","":"downstairs","session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1942026-07-20T22:57:58.682Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.754190596Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"":"downstairs","":"downstairs","session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1952026-07-20T22:57:58.682Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.754200715Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1962026-07-20T22:57:58.682Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.754214444Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"client":"0","":"downstairs","session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1972026-07-20T22:57:58.682Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.754224003Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"client":"1","":"downstairs","session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1982026-07-20T22:57:58.682Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.754238092Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"client":"2","":"downstairs","session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
1992026-07-20T22:57:58.682Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.754248491Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
2002026-07-20T22:57:58.682Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.75425867Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
2012026-07-20T22:57:58.682Z{"msg":"456d3519-b434-4cdf-83e3-9a6a413a5647 is now active with session: ddfb31c5-9662-4c69-bb26-7a85684284da","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.754285358Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
2022026-07-20T22:57:58.682Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.754294357Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"session_id":"ddfb31c5-9662-4c69-bb26-7a85684284da"}
2032026-07-20T22:57:58.682Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.754304126Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174}
2042026-07-20T22:57:58.682Z{"msg":"Activated sub_volume 456d3519-b434-4cdf-83e3-9a6a413a5647","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.754312636Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174}
2052026-07-20T22:57:58.682Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.754321515Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"task":"crutest"}
2062026-07-20T22:57:58.682ZDisk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2072026-07-20T22:57:58.682Z{"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.754507829Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"task":"crutest"}
2082026-07-20T22:57:58.698ZRead and Verify all blocks (0..50 range:false)
2092026-07-20T22:57:58.722Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.802844436Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H",----------------------------------------------------------------
2102026-07-20T22:57:58.722Z"pid":1174,"task":"crutest"}
2112026-07-20T22:57:58.722Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2122026-07-20T22:57:58.722Z{States:"msg":" Active Active ActiveCLIENT: Tests done. All submitted work has been ACK'd
2132026-07-20T22:57:58.722Z",JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2142026-07-20T22:57:58.722Z"v":0,"name":"crucible","level":30 1002 Acked Read 50," Donetime Done":" Done false
2152026-07-20T22:57:58.722Z2026-07-20T22:57:57.802876013Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H STATES DS:0 DS:1 DS:2 TOTAL
2162026-07-20T22:57:58.722Z" Sent ," 0 pid": 0 1174 0 0
2172026-07-20T22:57:58.722Z," Done task" 1 : 1 " 1 3
2182026-07-20T22:57:58.722Zcrutest" Skipped } 0
2192026-07-20T22:57:58.722Z 0 0 0
2202026-07-20T22:57:58.722Z Error 0 0 0 0
2212026-07-20T22:57:58.726ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2222026-07-20T22:57:58.726ZDownstairs last five completed: 1001 1000
2232026-07-20T22:57:58.726ZUpstairs last five completed: 1002 1001 1000
2242026-07-20T22:57:58.726Z{"msg":"CLIENT: Up:0 ds:1 act:3","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:57:57.802940408Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"task":"crutest"}
2252026-07-20T22:58:02.724Z----------------------------------------------------------------
2262026-07-20T22:58:02.724Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2272026-07-20T22:58:02.724ZDownstairs last five completed: 1003 1002 1001 1000
2282026-07-20T22:58:02.724ZUpstairs last five completed: 1003 1002 1001 1000
2292026-07-20T22:58:02.728Z{"msg":"CLIENT: Up:0 ds:0 act:3","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.80412834Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"task":"crutest"}
2302026-07-20T22:58:02.728Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.804196134Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174,"task":"crutest"}
2312026-07-20T22:58:02.728Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-20T22:58:01.804219272Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1174}
2322026-07-20T22:58:02.733ZRepair setup passed
2332026-07-20T22:58:02.737ZCopy the region for /var/tmp/test_up-build/8830
2342026-07-20T22:58:02.737Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2352026-07-20T22:58:02.740Z/var/tmp/bins/crutest fill -g 32 -q --key vG6yGA6icbhyoJRLwZ728fUSSYe4yhOxWoMKAwnMIcA= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2362026-07-20T22:58:02.756Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.836460008Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"task":"crutest"}
2372026-07-20T22:58:02.759Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.836829026Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"task":"crutest"}
2382026-07-20T22:58:02.760Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.837578722Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"task":"crutest"}
2392026-07-20T22:58:02.760Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.83783767Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180}
2402026-07-20T22:58:02.760Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a81b42ee266cbdd6371f13d7b3f8dd00b79d3cab\",\n git_commit_timestamp: \"2026-07-20T22:41:44.000000000Z\",\n git_branch: \"renovate/crate-rand-vulnerability\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.837869037Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180}
2412026-07-20T22:58:02.760Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.837882556Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180}
2422026-07-20T22:58:02.760Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.837895025Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180}
2432026-07-20T22:58:02.760Z{"msg":"Crucible 868943d5-75e1-495c-b88d-cd031ce98de4 has session id: 18ddb747-8b74-4509-979c-2ba1e659f01d","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.838337767Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2442026-07-20T22:58:02.760Z{"msg":"Upstairs opts: Upstairs UUID: 868943d5-75e1-495c-b88d-cd031ce98de4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.838377214Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2452026-07-20T22:58:02.760Z{"msg":"Crucible stats registered with UUID: 868943d5-75e1-495c-b88d-cd031ce98de4","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.838393542Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2462026-07-20T22:58:02.760Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-20T22:58:01.838403042Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2472026-07-20T22:58:02.760Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.838518652Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"":"io task","client":"1","":"downstairs","session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2482026-07-20T22:58:02.760Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.838557868Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"":"io task","client":"0","":"downstairs","session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2492026-07-20T22:58:02.760Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.838591535Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"":"io task","client":"2","":"downstairs","session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2502026-07-20T22:58:02.760Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.83876569Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"task":"crutest"}
2512026-07-20T22:58:02.760Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.838783349Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180}
2522026-07-20T22:58:02.760Z{"msg":"868943d5-75e1-495c-b88d-cd031ce98de4 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.838807327Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2532026-07-20T22:58:02.760Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.838821176Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"client":"0","":"downstairs","session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2542026-07-20T22:58:02.760Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.838831295Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"client":"1","":"downstairs","session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2552026-07-20T22:58:02.760Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.838845674Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"client":"2","":"downstairs","session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2562026-07-20T22:58:02.760Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.838856113Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"":"io task","client":"2","":"downstairs","session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2572026-07-20T22:58:02.760Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.838867142Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"":"io task","client":"1","":"downstairs","session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2582026-07-20T22:58:02.760Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.838882061Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"":"io task","client":"0","":"downstairs","session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2592026-07-20T22:58:02.760Z{"msg":"ds_connection connected from Ok(127.0.0.1:54115)","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.838950545Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"":"io task","client":"0","":"downstairs","session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2602026-07-20T22:58:02.760Z{"msg":"ds_connection connected from Ok(127.0.0.1:57553)","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.838977802Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"":"io task","client":"1","":"downstairs","session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2612026-07-20T22:58:02.760Z{"msg":"ds_connection connected from Ok(127.0.0.1:61523)","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.838990031Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"":"io task","client":"2","":"downstairs","session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2622026-07-20T22:58:02.764Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.839726978Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"client":"0","":"downstairs","session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2632026-07-20T22:58:02.764Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.839756266Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"client":"1","":"downstairs","session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2642026-07-20T22:58:02.764Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.839767795Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"client":"2","":"downstairs","session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2652026-07-20T22:58:02.764Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.8399387Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2662026-07-20T22:58:02.764Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.839990545Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2672026-07-20T22:58:02.764Z{"msg":"[0]R flush_numbers: [1499, 1499, 1499, 1499, 1499]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.840042381Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"":"downstairs","":"downstairs","session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2682026-07-20T22:58:02.764Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.8400579Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"":"downstairs","":"downstairs","session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2692026-07-20T22:58:02.764Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.840068169Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"":"downstairs","":"downstairs","session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2702026-07-20T22:58:02.764Z{"msg":"[1]R flush_numbers: [1499, 1499, 1499, 1499, 1499]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.840082778Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"":"downstairs","":"downstairs","session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2712026-07-20T22:58:02.764Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.840092857Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"":"downstairs","":"downstairs","session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2722026-07-20T22:58:02.764Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.840102996Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"":"downstairs","":"downstairs","session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2732026-07-20T22:58:02.764Z{"msg":"[2]R flush_numbers: [1499, 1499, 1499, 1499, 1499]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.840112875Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"":"downstairs","":"downstairs","session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2742026-07-20T22:58:02.764Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.840130434Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"":"downstairs","":"downstairs","session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2752026-07-20T22:58:02.764Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.840140323Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"":"downstairs","":"downstairs","session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2762026-07-20T22:58:02.764Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.840149852Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"":"downstairs","":"downstairs","session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2772026-07-20T22:58:02.764Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.84016736Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"":"downstairs","":"downstairs","session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2782026-07-20T22:58:02.764Z{"msg":"Next flush: 1500","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.84017703Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"":"downstairs","":"downstairs","session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2792026-07-20T22:58:02.764Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.840186509Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"":"downstairs","":"downstairs","session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2802026-07-20T22:58:02.764Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.840200617Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2812026-07-20T22:58:02.764Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.840209277Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"client":"0","":"downstairs","session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2822026-07-20T22:58:02.764Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.840218996Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"client":"1","":"downstairs","session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2832026-07-20T22:58:02.764Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.840232945Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"client":"2","":"downstairs","session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2842026-07-20T22:58:02.764Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.840242744Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2852026-07-20T22:58:02.764Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.840256513Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2862026-07-20T22:58:02.764Z{"msg":"Disk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2872026-07-20T22:58:02.764Z868943d5-75e1-495c-b88d-cd031ce98de4 is now active with session: 18ddb747-8b74-4509-979c-2ba1e659f01d","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.840276321Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2882026-07-20T22:58:02.765Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.84028505Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"session_id":"18ddb747-8b74-4509-979c-2ba1e659f01d"}
2892026-07-20T22:58:02.765Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.840298679Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180}
2902026-07-20T22:58:02.765Z{"msg":"Activated sub_volume 868943d5-75e1-495c-b88d-cd031ce98de4","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.840306858Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180}
2912026-07-20T22:58:02.765Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.840319587Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"task":"crutest"}
2922026-07-20T22:58:02.765ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
2932026-07-20T22:58:02.765Z{"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.840615572Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"task":"crutest"}
2942026-07-20T22:58:02.784ZRead and Verify all blocks (0..50 range:false)
2952026-07-20T22:58:02.808Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.887971323Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"task":"crutest"}
2962026-07-20T22:58:02.808Z{"msg":"CLIENT: Tests done. All submitted work has been ACK'd","v":0,"name":"crucible","level":30----------------------------------------------------------------
2972026-07-20T22:58:02.808Z,"time":" Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
2982026-07-20T22:58:02.808Z2026-07-20T22:58:01.887995971Z"States:,"hostname":" Activew-01KY0VTFMWMEWPCY0ZV7HPTC2H" Active, Active"
2992026-07-20T22:58:02.809Zpid":JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
3002026-07-20T22:58:02.809Z1180,"task":"crutest"}
3012026-07-20T22:58:02.809Z 1002 Acked Read 50 Done Done Done false
3022026-07-20T22:58:02.809Z STATES DS:0 DS:1 DS:2 TOTAL
3032026-07-20T22:58:02.809Z Sent 0 0 0 0
3042026-07-20T22:58:02.809Z Done 1 1 1 3
3052026-07-20T22:58:02.809Z Skipped 0 0 0 0
3062026-07-20T22:58:02.809Z Error 0 0 0 0
3072026-07-20T22:58:02.812ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
3082026-07-20T22:58:02.812ZDownstairs last five completed: 1001 1000
3092026-07-20T22:58:02.813ZUpstairs last five completed: 1002 1001 1000
3102026-07-20T22:58:02.813Z{"msg":"CLIENT: Up:0 ds:1 act:3","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:01.888065895Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"task":"crutest"}
3112026-07-20T22:58:06.809Z----------------------------------------------------------------
3122026-07-20T22:58:06.809Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
3132026-07-20T22:58:06.809ZDownstairs last five completed: 1003 1002 1001 1000
3142026-07-20T22:58:06.809ZUpstairs last five completed: 1003 1002 1001 1000
3152026-07-20T22:58:06.813Z{"msg":"CLIENT: Up:0 ds:0 act:3","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:05.889147316Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"task":"crutest"}
3162026-07-20T22:58:06.813Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:05.889198682Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180,"task":"crutest"}
3172026-07-20T22:58:06.813Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-20T22:58:05.88922397Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1180}
3182026-07-20T22:58:06.816ZRepair part 1 passed
3192026-07-20T22:58:06.816Z
3202026-07-20T22:58:06.817ZKill the current downstairs
3212026-07-20T22:58:06.859ZDownstairs 2 stopped
3222026-07-20T22:58:06.859Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
3232026-07-20T22:58:06.862ZNow put back the original so we have a mismatch
3242026-07-20T22:58:06.862Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
3252026-07-20T22:58:06.862ZRestart downstairs with old directory
3262026-07-20T22:58:06.875Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
3272026-07-20T22:58:06.886ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3282026-07-20T22:58:06.890Z 0 000-009 32 32 31 1500 1500 1499 F F F <---
3292026-07-20T22:58:06.890Z 1 010-019 32 32 31 1500 1500 1499 F F F <---
3302026-07-20T22:58:06.890Z 2 020-029 32 32 31 1500 1500 1499 F F F <---
3312026-07-20T22:58:06.890Z 3 030-039 32 32 31 1500 1500 1499 F F F <---
3322026-07-20T22:58:06.890Z 4 040-049 32 32 31 1500 1500 1499 F F F <---
3332026-07-20T22:58:06.890ZMax gen: 32, Max flush: 1500
3342026-07-20T22:58:07.111ZError: Difference in extent metadata found!
3352026-07-20T22:58:07.111Z
3362026-07-20T22:58:07.111ZStack backtrace:
3372026-07-20T22:58:07.111Z 0: anyhow::error::<impl anyhow::Error>::msg
3382026-07-20T22:58:07.111Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14
3392026-07-20T22:58:07.111Z 1: anyhow::__private::format_err
3402026-07-20T22:58:07.111Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13
3412026-07-20T22:58:07.111Z 2: crucible_downstairs::dump::dump_region
3422026-07-20T22:58:07.111Z at /work/oxidecomputer/crucible/downstairs/src/dump.rs:382:9
3432026-07-20T22:58:07.131Zdump test found error as expected
3442026-07-20T22:58:07.131Z
3452026-07-20T22:58:07.131Z
3462026-07-20T22:58:07.135Z/var/tmp/bins/crutest verify --range -g 33 -q --key vG6yGA6icbhyoJRLwZ728fUSSYe4yhOxWoMKAwnMIcA= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
3472026-07-20T22:58:07.151Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:06.231177402Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"task":"crutest"}
3482026-07-20T22:58:07.155Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:06.231516473Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"task":"crutest"}
3492026-07-20T22:58:07.155Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:06.232545424Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"task":"crutest"}
3502026-07-20T22:58:07.155Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:06.232787874Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188}
3512026-07-20T22:58:07.155Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a81b42ee266cbdd6371f13d7b3f8dd00b79d3cab\",\n git_commit_timestamp: \"2026-07-20T22:41:44.000000000Z\",\n git_branch: \"renovate/crate-rand-vulnerability\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:06.232816731Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188}
3522026-07-20T22:58:07.155Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:06.23282952Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188}
3532026-07-20T22:58:07.155Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:06.232841849Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188}
3542026-07-20T22:58:07.155Z{"msg":"Crucible 205e5ad5-24bd-4b56-85ed-e25649f150f7 has session id: f9a159f4-e244-47eb-b61f-07a0c9f6db4f","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:06.233351755Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3552026-07-20T22:58:07.155Z{"msg":"Upstairs opts: Upstairs UUID: 205e5ad5-24bd-4b56-85ed-e25649f150f7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:06.23340776Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3562026-07-20T22:58:07.155Z{"msg":"Crucible stats registered with UUID: 205e5ad5-24bd-4b56-85ed-e25649f150f7","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:06.233432168Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3572026-07-20T22:58:07.155Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-20T22:58:06.233446327Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3582026-07-20T22:58:07.155Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:06.233526Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3592026-07-20T22:58:07.155Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:06.233564227Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3602026-07-20T22:58:07.155Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:06.233579476Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3612026-07-20T22:58:07.155Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:06.233735632Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"task":"crutest"}
3622026-07-20T22:58:07.155Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:06.233776249Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188}
3632026-07-20T22:58:07.155Z{"msg":"205e5ad5-24bd-4b56-85ed-e25649f150f7 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:06.233851372Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3642026-07-20T22:58:07.155Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:06.233871461Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"client":"0","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3652026-07-20T22:58:07.155Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:06.23388578Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"client":"1","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3662026-07-20T22:58:07.155Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:06.233906478Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"client":"2","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3672026-07-20T22:58:07.156Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:06.233933945Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3682026-07-20T22:58:07.156Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:06.233955324Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3692026-07-20T22:58:07.156Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:06.233976352Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3702026-07-20T22:58:07.156Z{"msg":"ds_connection connected from Ok(127.0.0.1:50301)","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:06.23399828Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3712026-07-20T22:58:07.156Z{"msg":"ds_connection connected from Ok(127.0.0.1:40931)","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:06.234020508Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3722026-07-20T22:58:07.156Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-07-20T22:58:06.234050435Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3732026-07-20T22:58:07.156Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-07-20T22:58:06.234068324Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3742026-07-20T22:58:07.156Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-07-20T22:58:06.234087262Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3752026-07-20T22:58:07.156Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:06.234157336Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3762026-07-20T22:58:07.156Z{"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:06.234210022Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3772026-07-20T22:58:07.156Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:06.234605798Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"client":"0","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3782026-07-20T22:58:07.156Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:06.234628866Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"client":"1","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3792026-07-20T22:58:07.156Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:06.234793512Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3802026-07-20T22:58:07.156Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:06.23480888Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3812026-07-20T22:58:17.157Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.235597763Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3822026-07-20T22:58:17.157Z{"msg":"ds_connection connected from Ok(127.0.0.1:65226)","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.235695474Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3832026-07-20T22:58:17.163Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.237186997Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"client":"2","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3842026-07-20T22:58:17.163Z{"msg":"[0]R flush_numbers: [1500, 1500, 1500, 1500, 1500]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.237605541Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"downstairs","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3852026-07-20T22:58:17.163Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.237633678Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"downstairs","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3862026-07-20T22:58:17.163Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.237658996Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"downstairs","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3872026-07-20T22:58:17.163Z{"msg":"[1]R flush_numbers: [1500, 1500, 1500, 1500, 1500]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.237683574Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"downstairs","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3882026-07-20T22:58:17.163Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.237701202Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"downstairs","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3892026-07-20T22:58:17.163Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.23772555Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"downstairs","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3902026-07-20T22:58:17.163Z{"msg":"[2]R flush_numbers: [1499, 1499, 1499, 1499, 1499]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.237749918Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"downstairs","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3912026-07-20T22:58:17.163Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.237768017Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"downstairs","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3922026-07-20T22:58:17.163Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.237792065Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"downstairs","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3932026-07-20T22:58:17.163Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.237815273Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"downstairs","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3942026-07-20T22:58:17.163Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.237839621Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"downstairs","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3952026-07-20T22:58:17.163Z{"msg":"Next flush: 1501","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.237862119Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"downstairs","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3962026-07-20T22:58:17.163Z{"msg":"extent 0 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1499, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.237893176Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"mend","":"downstairs","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3972026-07-20T22:58:17.163Z{"msg":"extent:0 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1499, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.237926103Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"mend","":"downstairs","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3982026-07-20T22:58:17.163Z{"msg":"found dest for source 0 for extent at index 0 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.237952211Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"mend","":"downstairs","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
3992026-07-20T22:58:17.163Z{"msg":"extent 1 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1499, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.237983368Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"mend","":"downstairs","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
4002026-07-20T22:58:17.163Z{"msg":"extent:1 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1499, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.238014426Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"mend","":"downstairs","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
4012026-07-20T22:58:17.163Z{"msg":"found dest for source 0 for extent at index 1 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.238039923Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"mend","":"downstairs","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
4022026-07-20T22:58:17.163Z{"msg":"extent 2 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1499, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.238070761Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"mend","":"downstairs","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
4032026-07-20T22:58:17.163Z{"msg":"extent:2 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1499, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.238102248Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"mend","":"downstairs","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
4042026-07-20T22:58:17.163Z{"msg":"found dest for source 0 for extent at index 2 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.238127306Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"mend","":"downstairs","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
4052026-07-20T22:58:17.163Z{"msg":"extent 3 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1499, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.238152154Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"mend","":"downstairs","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
4062026-07-20T22:58:17.163Z{"msg":"extent:3 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1499, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.238182931Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"mend","":"downstairs","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
4072026-07-20T22:58:17.163Z{"msg":"found dest for source 0 for extent at index 3 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.238208209Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"mend","":"downstairs","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
4082026-07-20T22:58:17.163Z{"msg":"extent 4 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1499, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.238239246Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"mend","":"downstairs","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
4092026-07-20T22:58:17.163Z{"msg":"extent:4 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 1500, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 1499, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.238269864Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"mend","":"downstairs","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
4102026-07-20T22:58:17.163Z{"msg":"found dest for source 0 for extent at index 4 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.238295112Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"mend","":"downstairs","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
4112026-07-20T22:58:17.163Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.238329569Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"client":"0","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
4122026-07-20T22:58:17.163Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.238354726Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"client":"1","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
4132026-07-20T22:58:17.163Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.238374485Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"client":"2","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
4142026-07-20T22:58:17.163Z{"msg":"Full repair list: {ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.238411182Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"downstairs","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
4152026-07-20T22:58:17.163Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 1501, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 1501, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 1501, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 1501, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 1501, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.238557559Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"downstairs","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
4162026-07-20T22:58:17.164Z{"msg":"starting reconciliation a7e15b7d-9d55-4dde-a2d9-d7f90d079cbe: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.238577107Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"downstairs","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
4172026-07-20T22:58:17.164Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.238593156Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"client":"2","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
4182026-07-20T22:58:17.172Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.251140591Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"client":"2","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
4192026-07-20T22:58:17.175Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.25324443Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"client":"2","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
4202026-07-20T22:58:17.178Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.25511815Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"client":"2","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
4212026-07-20T22:58:17.178Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.257105729Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"client":"2","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
4222026-07-20T22:58:17.181Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.258771427Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"":"downstairs","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
4232026-07-20T22:58:17.181Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.258796434Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"client":"0","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
4242026-07-20T22:58:17.181Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.258809923Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"client":"1","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
4252026-07-20T22:58:17.181Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.258827592Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"client":"2","":"downstairs","session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
4262026-07-20T22:58:17.181Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.258840711Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
4272026-07-20T22:58:17.181Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.258857859Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
4282026-07-20T22:58:17.181Z{"msg":"205e5ad5-24bd-4b56-85ed-e25649f150f7 is now active with session: f9a159f4-e244-47eb-b61f-07a0c9f6db4f","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.258870358Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"session_id":"f9a159f4-e244-47eb-b61f-07a0c9f6db4f"}
4292026-07-20T22:58:17.181Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.258887927Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188}
4302026-07-20T22:58:17.181Z{"msg":"Activated sub_volume 205e5ad5-24bd-4b56-85ed-e25649f150f7","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.258903895Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188}
4312026-07-20T22:58:17.181Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.258955361Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"task":"crutest"}
4322026-07-20T22:58:17.181ZDisk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
4332026-07-20T22:58:17.181ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4342026-07-20T22:58:17.181ZRead and Verify all blocks (0..50 range:true)
4352026-07-20T22:58:17.205Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.283959688Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"task":"crutest"}
4362026-07-20T22:58:17.205Z{"msg":"Verify test completed","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.283980056Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"task":"crutest"}
4372026-07-20T22:58:17.205Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.283991155Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"task":"crutest"}
4382026-07-20T22:58:17.209Z{"msg":"CLIENT: Tests done. All submitted work has been ACK'd","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.284000584Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"task":"crutest"}
4392026-07-20T22:58:17.209Z----------------------------------------------------------------
4402026-07-20T22:58:17.209Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
4412026-07-20T22:58:17.209ZStates: Active Active Active
4422026-07-20T22:58:17.209ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4432026-07-20T22:58:17.209Z 1000 Acked Read 50 Done Done Done false
4442026-07-20T22:58:17.209Z STATES DS:0 DS:1 DS:2 TOTAL
4452026-07-20T22:58:17.209Z Sent 0 0 0 0
4462026-07-20T22:58:17.210Z Done 1 1 1 3
4472026-07-20T22:58:17.210Z Skipped 0 0 0 0
4482026-07-20T22:58:17.210Z Error 0 0 0 0
4492026-07-20T22:58:17.210ZLast Flush: None None None
4502026-07-20T22:58:17.210ZDownstairs last five completed:
4512026-07-20T22:58:17.210ZUpstairs last five completed: 1000
4522026-07-20T22:58:17.210Z{"msg":"CLIENT: Up:0 ds:1 act:3","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:16.284114944Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"task":"crutest"}
4532026-07-20T22:58:21.207Z----------------------------------------------------------------
4542026-07-20T22:58:21.207Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
4552026-07-20T22:58:21.207ZDownstairs last five completed: 1001 1000
4562026-07-20T22:58:21.207ZUpstairs last five completed: 1001 1000
4572026-07-20T22:58:21.210Z{"msg":"CLIENT: Up:0 ds:0 act:3","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:20.285560995Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"task":"crutest"}
4582026-07-20T22:58:21.210Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-07-20T22:58:20.285605671Z","hostname":"w-01KY0VTFMWMEWPCY0ZV7HPTC2H","pid":1188,"task":"crutest"}
4592026-07-20T22:58:21.213ZRepair part 2 passed
4602026-07-20T22:58:21.213Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4612026-07-20T22:58:21.225ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4622026-07-20T22:58:21.225Z 0 000-009 32 32 32 1500 1500 1500 F F F
4632026-07-20T22:58:21.228Z 1 010-019 32 32 32 1500 1500 1500 F F F
4642026-07-20T22:58:21.229Z 2 020-029 32 32 32 1500 1500 1500 F F F
4652026-07-20T22:58:21.229Z 3 030-039 32 32 32 1500 1500 1500 F F F
4662026-07-20T22:58:21.229Z 4 040-049 32 32 32 1500 1500 1500 F F F
4672026-07-20T22:58:21.229ZMax gen: 32, Max flush: 1500
4682026-07-20T22:58:21.229Zdump test passed
4692026-07-20T22:58:21.229Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
4702026-07-20T22:58:21.239Z Extent 1
4712026-07-20T22:58:21.241ZGEN 32 32 32
4722026-07-20T22:58:21.241ZFLUSH_ID 1500 1500 1500
4732026-07-20T22:58:21.241ZDIRTY
4742026-07-20T22:58:21.242Z
4752026-07-20T22:58:21.242ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
4762026-07-20T22:58:21.242Z010 A A A A A A
4772026-07-20T22:58:21.242Z011 A A A A A A
4782026-07-20T22:58:21.242Z012 A A A A A A
4792026-07-20T22:58:21.242Z013 A A A A A A
4802026-07-20T22:58:21.245Z014 A A A A A A
4812026-07-20T22:58:21.245Z015 A A A A A A
4822026-07-20T22:58:21.245Z016 A A A A A A
4832026-07-20T22:58:21.245Z017 A A A A A A
4842026-07-20T22:58:21.245Z018 A A A A A A
4852026-07-20T22:58:21.248Z019 A A A A A A
4862026-07-20T22:58:21.248Zdump extent test passed
4872026-07-20T22:58:21.248Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
4882026-07-20T22:58:21.257ZExtent 2 Block in extent 0 Actual block 20
4892026-07-20T22:58:21.257Z
4902026-07-20T22:58:21.260Z DATA SHA256 VER
4912026-07-20T22:58:21.260Z------ ---------------------------------------------------------------- ---
4922026-07-20T22:58:21.260Z 0 fe59bb796bd0a0fda5ca7c01cf36834e4ab795a98a6d6e5c4f4f4e05ec54dec4 A
4932026-07-20T22:58:21.261Z 1 fe59bb796bd0a0fda5ca7c01cf36834e4ab795a98a6d6e5c4f4f4e05ec54dec4 A
4942026-07-20T22:58:21.261Z 2 fe59bb796bd0a0fda5ca7c01cf36834e4ab795a98a6d6e5c4f4f4e05ec54dec4 A
4952026-07-20T22:58:21.261Z
4962026-07-20T22:58:21.261ZNONCES 0 1 2 DIFF
4972026-07-20T22:58:21.261Z------ ------------------------ ------------------------ ------------------------ -----
4982026-07-20T22:58:21.261Z 0 ab413382b6a0a130e00e4b9d ab413382b6a0a130e00e4b9d ab413382b6a0a130e00e4b9d
4992026-07-20T22:58:21.261Z
5002026-07-20T22:58:21.261Z TAGS 0 1 2 DIFF
5012026-07-20T22:58:21.261Z------ -------------------------------- -------------------------------- -------------------------------- -----
5022026-07-20T22:58:21.261Z 0 9a6d11535d749250b32ee6c048842ad8 9a6d11535d749250b32ee6c048842ad8 9a6d11535d749250b32ee6c048842ad8
5032026-07-20T22:58:21.261Z
5042026-07-20T22:58:21.261ZHASHES 0 1 2 DIFF
5052026-07-20T22:58:21.261Z------ ---------------- ---------------- ---------------- -----
5062026-07-20T22:58:21.261Z
5072026-07-20T22:58:21.265Zdump block test passed
5082026-07-20T22:58:21.265ZInitial upstairs tests have completed, stopping all downstairs
5092026-07-20T22:58:26.281ZCreating 4 larger downstairs regions
5102026-07-20T22:58:26.403ZStarting 4 downstairs
5112026-07-20T22:58:31.409Zdsc restarted at PID: 1201
5122026-07-20T22:58:31.469ZNow do the replace-reconcile test
5132026-07-20T22:58:31.482ZUsing 8840 for the replacement port
5142026-07-20T23:01:59.873ZCompleted test: replace-reconcile
5152026-07-20T23:01:59.877ZNow do the replace-before-active test
5162026-07-20T23:05:16.776ZCompleted test: replace-before-active
5172026-07-20T23:05:16.776ZAll tests have completed, stopping all downstairs
5182026-07-20T23:05:16.792Z
5192026-07-20T23:05:16.792ZAll Tests have passed
5202026-07-20T23:05:16.796Z9:10 Test duration
5212026-07-20T23:05:16.796Z
5222026-07-20T23:05:16.796Zreal 9:09.971896442
5232026-07-20T23:05:16.796Zuser 6:47.229274291
5242026-07-20T23:05:16.796Zsys 5.693797794
5252026-07-20T23:05:16.796Ztrap 0.187044854
5262026-07-20T23:05:16.796Ztflt 0.029038433
5272026-07-20T23:05:16.796Zdflt 0.012064315
5282026-07-20T23:05:16.796Zkflt 0.001327917
5292026-07-20T23:05:16.796Zlock 5:58:43.605127929
5302026-07-20T23:05:16.796Zslp 26:56.986989025
5312026-07-20T23:05:16.796Zlat 29.751750845
5322026-07-20T23:05:16.796Zstop 0.011821331
5332026-07-20T23:05:16.796Z+ echo 'test-up-encrypted ends'
5342026-07-20T23:05:16.796Ztest-up-encrypted ends
5352026-07-20T23:05:21.799Zprocess exited: duration 558735 ms, exit code 0
5362026-07-20T23:05:21.799Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
5372026-07-20T23:06:21.836Zexec warning: : stdout descriptor may be held open by a background process; giving up!
5382026-07-20T23:06:21.836Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
5392026-07-20T23:06:21.848Zfound 14 output files
5402026-07-20T23:06:21.848Zuploading: /tmp/test_up-build/dsc-out.txt (47384 bytes)
5412026-07-20T23:06:22.861Zuploaded: /tmp/test_up-build/dsc-out.txt
5422026-07-20T23:06:22.869Zuploading: /tmp/test_up-build/test_up_out.txt (22027990 bytes)
5432026-07-20T23:06:24.018Zuploaded: /tmp/test_up-build/test_up_out.txt
5442026-07-20T23:06:24.021Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (2509 bytes)
5452026-07-20T23:06:25.029Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
5462026-07-20T23:06:25.029Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (548341 bytes)
5472026-07-20T23:06:26.048Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
5482026-07-20T23:06:26.048Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (978654 bytes)
5492026-07-20T23:06:27.068Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
5502026-07-20T23:06:27.071Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (543070 bytes)
5512026-07-20T23:06:28.089Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
5522026-07-20T23:06:28.093Zuploading: /tmp/debug/df.txt (1270 bytes)
5532026-07-20T23:06:29.100Zuploaded: /tmp/debug/df.txt
5542026-07-20T23:06:29.100Zuploading: /tmp/debug/dtrace.txt (612533 bytes)
5552026-07-20T23:06:30.126Zuploaded: /tmp/debug/dtrace.txt
5562026-07-20T23:06:30.129Zuploading: /tmp/debug/iostat.txt (183116 bytes)
5572026-07-20T23:06:30.137Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 183116 -> 185492
5582026-07-20T23:06:31.144Zuploaded: /tmp/debug/iostat.txt
5592026-07-20T23:06:31.144Zuploading: /tmp/debug/mpstat.txt (447382 bytes)
5602026-07-20T23:06:31.156Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 447382 -> 453916
5612026-07-20T23:06:32.163Zuploaded: /tmp/debug/mpstat.txt
5622026-07-20T23:06:32.163Zuploading: /tmp/debug/paging.txt (78788 bytes)
5632026-07-20T23:06:32.171Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 78788 -> 80066
5642026-07-20T23:06:33.177Zuploaded: /tmp/debug/paging.txt
5652026-07-20T23:06:33.181Zuploading: /tmp/debug/prstat.txt (852714 bytes)
5662026-07-20T23:06:33.197Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 852714 -> 866941
5672026-07-20T23:06:34.204Zuploaded: /tmp/debug/prstat.txt
5682026-07-20T23:06:34.208Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
5692026-07-20T23:06:35.215Zuploaded: /tmp/debug/psrinfo.txt
5702026-07-20T23:06:35.220Zuploading: /tmp/debug/upstairs-info.txt (69139 bytes)
5712026-07-20T23:06:36.229Zuploaded: /tmp/debug/upstairs-info.txt