01JW9RGYXTV5QANAADEMVCENMJ: test-up-unencrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01JW9RHAXNYSKEZ0VQVYR1RMMB

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12025-05-27T21:11:28.995Zjob dependencies complete; ready to run (waiting for 14 m 35 s)
22025-05-27T21:12:41.207Zjob assigned to worker 01JW9SC4ZQ9YSQWP2B7N5AJD2Z [factory aws, i-02b6aadce43a851f8] (queued for 1 m 12 s)
32025-05-27T21:12:49.144Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42025-05-27T21:12:50.324Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52025-05-27T21:12:50.324Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62025-05-27T21:12:51.423Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72025-05-27T21:12:51.423Zdownloading input: /input/build/work/bins/crutest.gz
82025-05-27T21:12:52.613Zdownloaded input: /input/build/work/bins/crutest.gz
92025-05-27T21:12:52.613Zdownloading input: /input/build/work/bins/dsc.gz
102025-05-27T21:12:53.276Zdownloaded input: /input/build/work/bins/dsc.gz
112025-05-27T21:12:53.281Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122025-05-27T21:12:53.301Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132025-05-27T21:12:53.301Zdownloading input: /input/build/work/scripts/test_ds.sh
142025-05-27T21:12:53.326Zdownloaded input: /input/build/work/scripts/test_ds.sh
152025-05-27T21:12:53.326Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162025-05-27T21:12:53.347Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172025-05-27T21:12:53.347Zdownloading input: /input/build/work/scripts/test_repair.sh
182025-05-27T21:12:53.367Zdownloaded input: /input/build/work/scripts/test_repair.sh
192025-05-27T21:12:53.367Zdownloading input: /input/build/work/scripts/test_replay.sh
202025-05-27T21:12:53.390Zdownloaded input: /input/build/work/scripts/test_replay.sh
212025-05-27T21:12:53.391Zdownloading input: /input/build/work/scripts/test_up.sh
222025-05-27T21:12:53.412Zdownloaded input: /input/build/work/scripts/test_up.sh
232025-05-27T21:12:53.412Zdownloading input: /input/build/work/scripts/upstairs_info.d
242025-05-27T21:12:53.433Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252025-05-27T21:12:53.436Zdownloading input: /input/build/tmp/cargo-test-out.log
262025-05-27T21:12:54.099Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272025-05-27T21:12:54.100Zstarting task 0: "setup"
282025-05-27T21:12:54.103Z++ uname -s
292025-05-27T21:12:54.106Z+ kern=SunOS
302025-05-27T21:12:54.107Z+ build_user=build
312025-05-27T21:12:54.107Z+ build_uid=12345
322025-05-27T21:12:54.107Z+ work_dir=/work
332025-05-27T21:12:54.107Z+ input_dir=/input
342025-05-27T21:12:54.107Z+ [[ 0 == 12345 ]]
352025-05-27T21:12:54.107Z+ case "$kern" in
362025-05-27T21:12:54.107Z+ groupadd -g 12345 build
372025-05-27T21:12:54.110Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382025-05-27T21:12:56.116Z+ zfs create -o mountpoint=/work rpool/work
392025-05-27T21:12:56.314Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402025-05-27T21:12:56.317Z+ home_fs=zfs
412025-05-27T21:12:56.317Z+ [[ zfs == autofs ]]
422025-05-27T21:12:56.317Z+ mkdir -p /home/build
432025-05-27T21:12:56.321Z+ chown build:build /home/build /work
442025-05-27T21:12:58.320Z+ chmod 0700 /home/build /work
452025-05-27T21:12:58.324Zprocess exited: duration 4227 ms, exit code 0
 
462025-05-27T21:12:58.330Zstarting task 1: "authentication"
472025-05-27T21:12:58.348Zprocess exited: duration 18 ms, exit code 0
 
482025-05-27T21:12:58.357Zstarting task 2: "build"
492025-05-27T21:12:58.361Z+ banner cores
502025-05-27T21:12:58.364Z
512025-05-27T21:12:58.364Z #### #### ##### ###### ####
522025-05-27T21:12:58.364Z # # # # # # # #
532025-05-27T21:12:58.364Z # # # # # ##### ####
542025-05-27T21:12:58.364Z # # # ##### # #
552025-05-27T21:12:58.364Z # # # # # # # # #
562025-05-27T21:12:58.364Z #### #### # # ###### ####
572025-05-27T21:12:58.364Z
582025-05-27T21:12:58.364Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
592025-05-27T21:12:58.372Z+ echo 'input bins dir contains:'
602025-05-27T21:12:58.372Zinput bins dir contains:
612025-05-27T21:12:58.375Z+ ls -ltr /input/build/work/bins
622025-05-27T21:12:58.375Ztotal 798184
632025-05-27T21:12:58.375Z-rw-r--r-- 1 root root 116152933 May 27 21:12 crucible-downstairs.gz
642025-05-27T21:12:58.375Z-rw-r--r-- 1 root root 109360001 May 27 21:12 crucible-hammer.gz
652025-05-27T21:12:58.375Z-rw-r--r-- 1 root root 118453644 May 27 21:12 crutest.gz
662025-05-27T21:12:58.375Z-rw-r--r-- 1 root root 64443062 May 27 21:12 dsc.gz
672025-05-27T21:12:58.375Z+ banner unpack
682025-05-27T21:12:58.380Z
692025-05-27T21:12:58.380Z # # # # ##### ## #### # #
702025-05-27T21:12:58.380Z # # ## # # # # # # # # #
712025-05-27T21:12:58.380Z # # # # # # # # # # ####
722025-05-27T21:12:58.380Z # # # # # ##### ###### # # #
732025-05-27T21:12:58.380Z # # # ## # # # # # # #
742025-05-27T21:12:58.380Z #### # # # # # #### # #
752025-05-27T21:12:58.380Z
762025-05-27T21:12:58.380Z+ mkdir -p /var/tmp/bins
772025-05-27T21:12:58.380Z+ for t in "$input/bins/"*.gz
782025-05-27T21:12:58.383Z++ basename /input/build/work/bins/crucible-downstairs.gz
792025-05-27T21:12:58.383Z+ b=crucible-downstairs.gz
802025-05-27T21:12:58.384Z+ b=crucible-downstairs
812025-05-27T21:12:58.384Z+ gunzip
822025-05-27T21:13:00.780Z+ chmod +x /var/tmp/bins/crucible-downstairs
832025-05-27T21:13:00.784Z+ for t in "$input/bins/"*.gz
842025-05-27T21:13:00.784Z++ basename /input/build/work/bins/crucible-hammer.gz
852025-05-27T21:13:00.787Z+ b=crucible-hammer.gz
862025-05-27T21:13:00.787Z+ b=crucible-hammer
872025-05-27T21:13:00.787Z+ gunzip
882025-05-27T21:13:03.031Z+ chmod +x /var/tmp/bins/crucible-hammer
892025-05-27T21:13:03.034Z+ for t in "$input/bins/"*.gz
902025-05-27T21:13:03.034Z++ basename /input/build/work/bins/crutest.gz
912025-05-27T21:13:03.037Z+ b=crutest.gz
922025-05-27T21:13:03.037Z+ b=crutest
932025-05-27T21:13:03.037Z+ gunzip
942025-05-27T21:13:05.558Z+ chmod +x /var/tmp/bins/crutest
952025-05-27T21:13:05.562Z+ for t in "$input/bins/"*.gz
962025-05-27T21:13:05.562Z++ basename /input/build/work/bins/dsc.gz
972025-05-27T21:13:05.565Z+ b=dsc.gz
982025-05-27T21:13:05.565Z+ b=dsc
992025-05-27T21:13:05.565Z+ gunzip
1002025-05-27T21:13:06.865Z+ chmod +x /var/tmp/bins/dsc
1012025-05-27T21:13:06.868Z+ export BINDIR=/var/tmp/bins
1022025-05-27T21:13:06.868Z+ BINDIR=/var/tmp/bins
1032025-05-27T21:13:06.869Z+ export RUST_BACKTRACE=1
1042025-05-27T21:13:06.869Z+ RUST_BACKTRACE=1
1052025-05-27T21:13:06.869Z+ jobpid=1160
1062025-05-27T21:13:06.869Z+ echo 'Setup debug logging'
1072025-05-27T21:13:06.869ZSetup debug logging
1082025-05-27T21:13:06.869Z+ mkdir /tmp/debug
1092025-05-27T21:13:06.869Z+ sleep 7200
1102025-05-27T21:13:06.872Z+ psrinfo -v
1112025-05-27T21:13:06.876Z+ df -h
1122025-05-27T21:13:06.883Z+ prstat -d d -mLc 1
1132025-05-27T21:13:06.886Z+ iostat -T d -xn 1
1142025-05-27T21:13:06.886Z+ mpstat -T d 1
1152025-05-27T21:13:06.886Z+ vmstat -T d -p 1
1162025-05-27T21:13:06.886Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1172025-05-27T21:13:06.890Z+ banner test_up_unencrypted
1182025-05-27T21:13:06.890Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1192025-05-27T21:13:06.890Z
1202025-05-27T21:13:06.890Z ##### ###### #### ##### # # ##### # # # #
1212025-05-27T21:13:06.890Z # # # # # # # # # # ## #
1222025-05-27T21:13:06.890Z # ##### #### # # # # # # # # # #
1232025-05-27T21:13:06.890Z # # # # # # ##### # # # # #
1242025-05-27T21:13:06.890Z # # # # # # # # # # # ##
1252025-05-27T21:13:06.890Z # ###### #### # ####### #### # ####### #### # #
1262025-05-27T21:13:06.890Z
1272025-05-27T21:13:06.890Z+ ptime -m bash /input/build/work/scripts/test_up.sh -N unencrypted
1282025-05-27T21:13:06.902Z/input/build/work
1292025-05-27T21:13:06.914ZTurn off color for downstairs dump
1302025-05-27T21:13:06.918Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1312025-05-27T21:13:06.918ZCreating 3 downstairs regions
1322025-05-27T21:13:07.244ZStarting 3 downstairs
1332025-05-27T21:13:07.248Zdsc started at PID: 1205
1342025-05-27T21:13:12.428ZDisable automatic restart on all downstairs
1352025-05-27T21:13:12.483Z
1362025-05-27T21:13:12.483ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1372025-05-27T21:13:12.486ZRunning test: span
1382025-05-27T21:13:16.636ZCompleted test: span
1392025-05-27T21:13:21.643ZRunning test: big
1402025-05-27T21:13:25.791ZCompleted test: big
1412025-05-27T21:13:30.798ZRunning test: dep
1422025-05-27T21:13:43.791ZCompleted test: dep
1432025-05-27T21:13:48.798ZRunning test: balloon
1442025-05-27T21:13:58.609ZCompleted test: balloon
1452025-05-27T21:14:03.616ZRunning test: deactivate
1462025-05-27T21:14:57.828ZCompleted test: deactivate
1472025-05-27T21:15:02.835ZRunning hammer
1482025-05-27T21:15:05.830ZRun repair tests
1492025-05-27T21:15:05.834Z/var/tmp/bins/crutest fill -g 31 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1502025-05-27T21:15:05.891Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.444560887Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"task":"crutest"}
1512025-05-27T21:15:05.895Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.44525404Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"task":"crutest"}
1522025-05-27T21:15:05.895Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.446896993Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"task":"crutest"}
1532025-05-27T21:15:05.895Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.447095259Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242}
1542025-05-27T21:15:05.895Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"dd892d47062b1dcb23800b8bbb5d2946e1dd1cb7\",\n git_commit_timestamp: \"2025-05-27T20:18:55.000000000Z\",\n git_branch: \"alan/agent-needs-workers\",\n rustc_semver: \"1.86.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"05f9846f893b09a1be1fc8560e33fc3c815cfecb\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.44713255Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242}
1552025-05-27T21:15:05.895Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.447146261Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242}
1562025-05-27T21:15:05.895Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.447167922Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242}
1572025-05-27T21:15:05.895Z{"msg":"Crucible 26c195d1-2259-44b7-a88a-b49b70b46d59 has session id: d3836926-66a5-4536-b404-aa01474a508a","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.447793602Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1582025-05-27T21:15:05.895Z{"msg":"Upstairs opts: Upstairs UUID: 26c195d1-2259-44b7-a88a-b49b70b46d59, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.447847593Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1592025-05-27T21:15:05.895Z{"msg":"Crucible stats registered with UUID: 26c195d1-2259-44b7-a88a-b49b70b46d59","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.447872774Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1602025-05-27T21:15:05.895Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-05-27T21:15:05.447910425Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1612025-05-27T21:15:05.900Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.448819375Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"task":"crutest"}
1622025-05-27T21:15:05.900Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.448871626Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242}
1632025-05-27T21:15:05.900Z{"msg":"26c195d1-2259-44b7-a88a-b49b70b46d59 active request set","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.448950669Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1642025-05-27T21:15:05.900Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.44898318Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"client":"0","":"downstairs","session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1652025-05-27T21:15:05.900Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.44900717Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"client":"0","":"downstairs","session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1662025-05-27T21:15:05.900Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.449021451Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"client":"1","":"downstairs","session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1672025-05-27T21:15:05.900Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.449035382Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"client":"1","":"downstairs","session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1682025-05-27T21:15:05.901Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.449047772Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"client":"2","":"downstairs","session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1692025-05-27T21:15:05.901Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.449062392Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"client":"2","":"downstairs","session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1702025-05-27T21:15:05.901Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.449074943Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"":"io task","client":"2","":"downstairs","session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1712025-05-27T21:15:05.901Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.449096084Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"":"io task","client":"0","":"downstairs","session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1722025-05-27T21:15:05.901Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.449132025Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"":"io task","client":"1","":"downstairs","session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1732025-05-27T21:15:05.901Z{"msg":"ds_connection connected from Ok(127.0.0.1:57925)","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.449159595Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"":"io task","client":"2","":"downstairs","session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1742025-05-27T21:15:05.901Z{"msg":"ds_connection connected from Ok(127.0.0.1:38594)","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.449193676Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"":"io task","client":"0","":"downstairs","session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1752025-05-27T21:15:05.901Z{"msg":"ds_connection connected from Ok(127.0.0.1:38009)","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.449470865Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"":"io task","client":"1","":"downstairs","session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1762025-05-27T21:15:05.901Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.449883539Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"client":"0","":"downstairs","session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1772025-05-27T21:15:05.901Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.44992765Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"client":"1","":"downstairs","session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1782025-05-27T21:15:05.901Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.450100926Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"client":"2","":"downstairs","session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1792025-05-27T21:15:05.901Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.45054326Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"client":"0","":"downstairs","session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1802025-05-27T21:15:05.901Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.450579201Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"client":"1","":"downstairs","session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1812025-05-27T21:15:05.901Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.450595342Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"client":"2","":"downstairs","session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1822025-05-27T21:15:05.901Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.450962064Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1832025-05-27T21:15:05.901Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.451002855Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1842025-05-27T21:15:05.901Z{"msg":"[0]R flush_numbers: [1496, 1496, 1496, 1496, 1496]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.451059296Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"":"downstairs","":"downstairs","session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1852025-05-27T21:15:05.901Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.451095878Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"":"downstairs","":"downstairs","session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1862025-05-27T21:15:05.901Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.451115168Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"":"downstairs","":"downstairs","session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1872025-05-27T21:15:05.901Z{"msg":"[1]R flush_numbers: [1496, 1496, 1496, 1496, 1496]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.451150639Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"":"downstairs","":"downstairs","session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1882025-05-27T21:15:05.901Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.45116653Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"":"downstairs","":"downstairs","session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1892025-05-27T21:15:05.901Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.451202941Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"":"downstairs","":"downstairs","session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1902025-05-27T21:15:05.901Z{"msg":"[2]R flush_numbers: [1496, 1496, 1496, 1496, 1496]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.451223792Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"":"downstairs","":"downstairs","session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1912025-05-27T21:15:05.901Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
1922025-05-27T21:15:05.901Z,"time":"2025-05-27T21:15:05.451255493Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"":"downstairs","":"downstairs","session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1932025-05-27T21:15:05.901Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.451289754Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"":"downstairs","":"downstairs","session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1942025-05-27T21:15:05.901Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.451317385Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"":"downstairs","":"downstairs","session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1952025-05-27T21:15:05.901Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.451343636Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"":"downstairs","":"downstairs","session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1962025-05-27T21:15:05.901Z{"msg":"Next flush: 1497","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.451374067Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"":"downstairs","":"downstairs","session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1972025-05-27T21:15:05.901Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.451396467Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"":"downstairs","":"downstairs","session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1982025-05-27T21:15:05.901Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.451426298Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
1992025-05-27T21:15:05.901Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.451446189Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"client":"0","":"downstairs","session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
2002025-05-27T21:15:05.901Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.45147414Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"client":"1","":"downstairs","session_id":"d3836926-66a5-4536-b404-aa01474a508a"}Disk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2012025-05-27T21:15:05.901Z
2022025-05-27T21:15:05.901Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.451500711Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"client":"2","":"downstairs","session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
2032025-05-27T21:15:05.901Z{"msg":"Fill test
2042025-05-27T21:15:05.901ZAll required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.451528202Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
2052025-05-27T21:15:05.901Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.451548852Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
2062025-05-27T21:15:05.901Z{"msg":"26c195d1-2259-44b7-a88a-b49b70b46d59 is now active with session: d3836926-66a5-4536-b404-aa01474a508a","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.451577743Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
2072025-05-27T21:15:05.902Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.451598124Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242,"session_id":"d3836926-66a5-4536-b404-aa01474a508a"}
2082025-05-27T21:15:05.902Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.451623945Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242}
2092025-05-27T21:15:05.902Z{"msg":"Activated sub_volume 26c195d1-2259-44b7-a88a-b49b70b46d59","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:05.451642135Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242}
2102025-05-27T21:15:05.905ZRead and Verify all blocks (0..50 range:false)
2112025-05-27T21:15:05.910ZWrote out file "/var/tmp/test_up-build/verify_file"
2122025-05-27T21:15:05.910ZCLIENT: Tests done. All submitted work has been ACK'd
2132025-05-27T21:15:05.913Z----------------------------------------------------------------
2142025-05-27T21:15:05.913Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2152025-05-27T21:15:05.913ZStates: Active Active Active
2162025-05-27T21:15:05.913ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2172025-05-27T21:15:05.913Z 1002 Acked Read 50 Done Done Done false
2182025-05-27T21:15:05.913Z STATES DS:0 DS:1 DS:2 TOTAL
2192025-05-27T21:15:05.913Z Sent 0 0 0 0
2202025-05-27T21:15:05.913Z Done 1 1 1 3
2212025-05-27T21:15:05.913Z Skipped 0 0 0 0
2222025-05-27T21:15:05.914Z Error 0 0 0 0
2232025-05-27T21:15:05.914ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2242025-05-27T21:15:05.914ZDownstairs last five completed: 1001 1000
2252025-05-27T21:15:05.914ZUpstairs last five completed: 1002 1001 1000
2262025-05-27T21:15:05.914ZCLIENT: Up:0 ds:1 act:3
2272025-05-27T21:15:09.912Z----------------------------------------------------------------
2282025-05-27T21:15:09.912Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2292025-05-27T21:15:09.912ZDownstairs last five completed: 1003 1002 1001 1000
2302025-05-27T21:15:09.912ZUpstairs last five completed: 1003 1002 1001 1000
2312025-05-27T21:15:09.912ZCLIENT: Up:0 ds:0 act:3
2322025-05-27T21:15:09.912ZCLIENT: All crucible jobs finished, exiting program
2332025-05-27T21:15:09.915Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-05-27T21:15:09.46462685Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1242}
2342025-05-27T21:15:09.921ZRepair setup passed
2352025-05-27T21:15:09.925ZCopy the region for /var/tmp/test_up-build/8830
2362025-05-27T21:15:09.928Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2372025-05-27T21:15:09.932Z/var/tmp/bins/crutest fill -g 32 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2382025-05-27T21:15:09.986Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.539402465Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"task":"crutest"}
2392025-05-27T21:15:09.990Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.539913111Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"task":"crutest"}
2402025-05-27T21:15:09.990Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.541621566Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"task":"crutest"}
2412025-05-27T21:15:09.990Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.541817492Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248}
2422025-05-27T21:15:09.990Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"dd892d47062b1dcb23800b8bbb5d2946e1dd1cb7\",\n git_commit_timestamp: \"2025-05-27T20:18:55.000000000Z\",\n git_branch: \"alan/agent-needs-workers\",\n rustc_semver: \"1.86.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"05f9846f893b09a1be1fc8560e33fc3c815cfecb\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.541862604Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248}
2432025-05-27T21:15:09.990Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.541888085Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248}
2442025-05-27T21:15:09.990Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.541906085Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248}
2452025-05-27T21:15:09.990Z{"msg":"Crucible 0f071452-6c15-4edb-b0b3-a7f53aff5bec has session id: 39e62802-e600-4c10-8860-1445b53200fe","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.542602148Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2462025-05-27T21:15:09.990Z{"msg":"Upstairs opts: Upstairs UUID: 0f071452-6c15-4edb-b0b3-a7f53aff5bec, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.542655189Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2472025-05-27T21:15:09.990Z{"msg":"Crucible stats registered with UUID: 0f071452-6c15-4edb-b0b3-a7f53aff5bec","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.54267541Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2482025-05-27T21:15:09.990Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-05-27T21:15:09.542695781Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2492025-05-27T21:15:09.996Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.543538988Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"task":"crutest"}
2502025-05-27T21:15:09.996Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.543575989Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248}
2512025-05-27T21:15:09.996Z{"msg":"0f071452-6c15-4edb-b0b3-a7f53aff5bec active request set","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.543669712Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2522025-05-27T21:15:09.996Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.543714213Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"client":"0","":"downstairs","session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2532025-05-27T21:15:09.996Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.543766445Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"client":"0","":"downstairs","session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2542025-05-27T21:15:09.996Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.543800796Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"client":"1","":"downstairs","session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2552025-05-27T21:15:09.996Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.543843067Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"client":"1","":"downstairs","session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2562025-05-27T21:15:09.997Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.543874979Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"client":"2","":"downstairs","session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2572025-05-27T21:15:09.997Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.54391263Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"client":"2","":"downstairs","session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2582025-05-27T21:15:09.997Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.544129107Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2592025-05-27T21:15:09.997Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.544179978Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2602025-05-27T21:15:09.997Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.544207379Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2612025-05-27T21:15:09.997Z{"msg":"ds_connection connected from Ok(127.0.0.1:56892)","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.54424722Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2622025-05-27T21:15:09.997Z{"msg":"ds_connection connected from Ok(127.0.0.1:39186)","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.544276032Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2632025-05-27T21:15:09.997Z{"msg":"ds_connection connected from Ok(127.0.0.1:37869)","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.544311693Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2642025-05-27T21:15:09.997Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.544670264Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"client":"2","":"downstairs","session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2652025-05-27T21:15:09.997Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.544709975Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"client":"0","":"downstairs","session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2662025-05-27T21:15:09.997Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.544734846Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"client":"1","":"downstairs","session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2672025-05-27T21:15:09.997Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.545405348Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"client":"1","":"downstairs","session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2682025-05-27T21:15:09.997Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.545441979Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"client":"2","":"downstairs","session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2692025-05-27T21:15:09.997Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.54545739Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"client":"0","":"downstairs","session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2702025-05-27T21:15:09.997Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.545854122Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2712025-05-27T21:15:09.997Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.545899064Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2722025-05-27T21:15:09.997Z{"msg":"[0]R flush_numbers: [1497, 1497, 1497, 1497, 1497]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.545918644Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"":"downstairs","":"downstairs","session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2732025-05-27T21:15:09.997Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.545942215Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"":"downstairs","":"downstairs","session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2742025-05-27T21:15:09.997Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.545955815Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"":"downstairs","":"downstairs","session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2752025-05-27T21:15:09.997Z{"msg":"[1]R flush_numbers: [1497, 1497, 1497, 1497, 1497]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.545976996Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"":"downstairs","":"downstairs","session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2762025-05-27T21:15:09.997Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.545991347Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"":"downstairs","":"downstairs","session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2772025-05-27T21:15:09.997Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.546014887Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"":"downstairs","":"downstairs","session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2782025-05-27T21:15:09.997Z{"msg":"[2]R flush_numbers: [1497, 1497, 1497, 1497, 1497]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.546052049Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"":"downstairs","":"downstairs","session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2792025-05-27T21:15:09.997Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.546071619Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"":"downstairs","":"downstairs","session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2802025-05-27T21:15:09.997Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.54609712Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"":"downstairs","":"downstairs","session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2812025-05-27T21:15:09.997Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.546119221Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"":"downstairs","":"downstairs","session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2822025-05-27T21:15:09.997Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.546153282Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"":"downstairs","":"downstairs","session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2832025-05-27T21:15:09.997ZWait for a query_work_queue command to finish before sending IO
2842025-05-27T21:15:09.997Z{"msg":"Next flush: 1498","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.546192773Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"":"downstairs","":"downstairs","session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2852025-05-27T21:15:09.997Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.546232084Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"":"downstairs","":"downstairs","session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2862025-05-27T21:15:09.997Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.546250135Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2872025-05-27T21:15:09.997Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.546269356Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"client":"0","":"downstairs","session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2882025-05-27T21:15:09.997Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.546290956Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"client":"1","":"downstairs","session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2892025-05-27T21:15:09.997Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.546325567Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"client":"2","":"downstairs","session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2902025-05-27T21:15:09.997Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.546354248Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2912025-05-27T21:15:09.997Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.546368269Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2922025-05-27T21:15:09.997Z{"msg":"0f071452-6c15-4edb-b0b3-a7f53aff5bec is now active with session: 39e62802-e600-4c10-8860-1445b53200fe","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.54640046Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2932025-05-27T21:15:09.997Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.546431541Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248,"session_id":"39e62802-e600-4c10-8860-1445b53200fe"}
2942025-05-27T21:15:09.997Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.546458162Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248}
2952025-05-27T21:15:09.997Z{"msg":"Activated sub_volume 0f071452-6c15-4edb-b0b3-a7f53aff5bec","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:09.546476042Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248}
2962025-05-27T21:15:09.998ZDisk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2972025-05-27T21:15:09.998ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
2982025-05-27T21:15:09.998ZFill test
2992025-05-27T21:15:10.001ZRead and Verify all blocks (0..50 range:false)
3002025-05-27T21:15:10.005ZWrote out file "/var/tmp/test_up-build/verify_file"
3012025-05-27T21:15:10.005ZCLIENT: Tests done. All submitted work has been ACK'd
3022025-05-27T21:15:10.005Z----------------------------------------------------------------
3032025-05-27T21:15:10.005Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
3042025-05-27T21:15:10.005ZStates: Active Active Active
3052025-05-27T21:15:10.005ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
3062025-05-27T21:15:10.005Z 1002 Acked Read 50 Done Done Done false
3072025-05-27T21:15:10.005Z STATES DS:0 DS:1 DS:2 TOTAL
3082025-05-27T21:15:10.005Z Sent 0 0 0 0
3092025-05-27T21:15:10.005Z Done 1 1 1 3
3102025-05-27T21:15:10.005Z Skipped 0 0 0 0
3112025-05-27T21:15:10.005Z Error 0 0 0 0
3122025-05-27T21:15:10.005ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
3132025-05-27T21:15:10.005ZDownstairs last five completed: 1001 1000
3142025-05-27T21:15:10.005ZUpstairs last five completed: 1002 1001 1000
3152025-05-27T21:15:10.005ZCLIENT: Up:0 ds:1 act:3
3162025-05-27T21:15:14.006Z----------------------------------------------------------------
3172025-05-27T21:15:14.006Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
3182025-05-27T21:15:14.006ZDownstairs last five completed: 1003 1002 1001 1000
3192025-05-27T21:15:14.006ZUpstairs last five completed: 1003 1002 1001 1000
3202025-05-27T21:15:14.006ZCLIENT: Up:0 ds:0 act:3
3212025-05-27T21:15:14.006ZCLIENT: All crucible jobs finished, exiting program
3222025-05-27T21:15:14.010Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-05-27T21:15:13.559412555Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1248}
3232025-05-27T21:15:14.013ZRepair part 1 passed
3242025-05-27T21:15:14.013Z
3252025-05-27T21:15:14.013ZKill the current downstairs
3262025-05-27T21:15:14.183ZDownstairs 2 stopped
3272025-05-27T21:15:14.183Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
3282025-05-27T21:15:14.187ZNow put back the original so we have a mismatch
3292025-05-27T21:15:14.187Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
3302025-05-27T21:15:14.190ZRestart downstairs with old directory
3312025-05-27T21:15:14.245Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
3322025-05-27T21:15:14.262ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3332025-05-27T21:15:14.266Z 0 000-009 32 32 31 1498 1498 1497 F F F <---
3342025-05-27T21:15:14.266Z 1 010-019 32 32 31 1498 1498 1497 F F F <---
3352025-05-27T21:15:14.266Z 2 020-029 32 32 31 1498 1498 1497 F F F <---
3362025-05-27T21:15:14.266Z 3 030-039 32 32 31 1498 1498 1497 F F F <---
3372025-05-27T21:15:14.266Z 4 040-049 32 32 31 1498 1498 1497 F F F <---
3382025-05-27T21:15:14.266ZMax gen: 32, Max flush: 1498
3392025-05-27T21:15:14.480ZError: Difference in extent metadata found!
3402025-05-27T21:15:14.480Z
3412025-05-27T21:15:14.480ZStack backtrace:
3422025-05-27T21:15:14.480Z 0: anyhow::error::<impl anyhow::Error>::msg
3432025-05-27T21:15:14.480Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.97/src/backtrace.rs:27:14
3442025-05-27T21:15:14.480Z 1: anyhow::__private::format_err
3452025-05-27T21:15:14.480Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.97/src/lib.rs:692:13
3462025-05-27T21:15:14.499Zdump test found error as expected
3472025-05-27T21:15:14.499Z
3482025-05-27T21:15:14.499Z
3492025-05-27T21:15:14.505Z/var/tmp/bins/crutest verify --range -g 33 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
3502025-05-27T21:15:14.557Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:14.109866975Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"task":"crutest"}
3512025-05-27T21:15:14.571Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:14.110421493Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"task":"crutest"}
3522025-05-27T21:15:14.571Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:14.112090687Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"task":"crutest"}
3532025-05-27T21:15:14.571Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:14.112307734Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256}
3542025-05-27T21:15:14.571Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"dd892d47062b1dcb23800b8bbb5d2946e1dd1cb7\",\n git_commit_timestamp: \"2025-05-27T20:18:55.000000000Z\",\n git_branch: \"alan/agent-needs-workers\",\n rustc_semver: \"1.86.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"05f9846f893b09a1be1fc8560e33fc3c815cfecb\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:14.112386496Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256}
3552025-05-27T21:15:14.571Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:14.112425888Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256}
3562025-05-27T21:15:14.571Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:14.112451238Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256}
3572025-05-27T21:15:14.571Z{"msg":"Crucible af9477eb-51d6-42dc-840b-b237a825ab89 has session id: 274033b9-91b0-4d8e-8ffd-2a0e1e3871ec","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:14.112974675Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3582025-05-27T21:15:14.571Z{"msg":"Upstairs opts: Upstairs UUID: af9477eb-51d6-42dc-840b-b237a825ab89, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:14.113049717Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3592025-05-27T21:15:14.571Z{"msg":"Crucible stats registered with UUID: af9477eb-51d6-42dc-840b-b237a825ab89","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:14.113080669Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3602025-05-27T21:15:14.572Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-05-27T21:15:14.113098439Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3612025-05-27T21:15:14.575Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:14.113954126Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"task":"crutest"}
3622025-05-27T21:15:14.576Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:14.114010368Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256}
3632025-05-27T21:15:14.576Z{"msg":"af9477eb-51d6-42dc-840b-b237a825ab89 active request set","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:14.114145413Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3642025-05-27T21:15:14.576Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:14.114185344Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"client":"0","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3652025-05-27T21:15:14.576Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:14.114247086Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"client":"0","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3662025-05-27T21:15:14.576Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:14.114276117Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"client":"1","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3672025-05-27T21:15:14.576Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:14.114315758Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"client":"1","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3682025-05-27T21:15:14.576Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:14.114340429Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"client":"2","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3692025-05-27T21:15:14.576Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:14.11437495Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"client":"2","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3702025-05-27T21:15:14.576Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:14.114405511Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3712025-05-27T21:15:14.576Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:14.114429432Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3722025-05-27T21:15:14.576Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:14.114467823Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3732025-05-27T21:15:14.576Z{"msg":"ds_connection connected from Ok(127.0.0.1:56456)","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:14.114516375Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3742025-05-27T21:15:14.576Z{"msg":"ds_connection connected from Ok(127.0.0.1:34278)","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:14.114553956Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3752025-05-27T21:15:14.576Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-05-27T21:15:14.114581787Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3762025-05-27T21:15:14.576Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-05-27T21:15:14.11469457Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3772025-05-27T21:15:14.576Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-05-27T21:15:14.114744382Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3782025-05-27T21:15:14.576Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:14.114767693Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3792025-05-27T21:15:14.576Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:14.115031371Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"client":"0","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3802025-05-27T21:15:14.576Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:14.115084503Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"client":"1","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3812025-05-27T21:15:14.576Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:14.115693473Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"client":"0","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3822025-05-27T21:15:14.576Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:14.115763315Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"client":"1","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3832025-05-27T21:15:14.576Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:14.116065345Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3842025-05-27T21:15:14.576Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:14.116145297Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3852025-05-27T21:15:24.564Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.116363471Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3862025-05-27T21:15:24.569Z{"msg":"ds_connection connected from Ok(127.0.0.1:41162)","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.116739954Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3872025-05-27T21:15:24.569Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.117639483Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"client":"2","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3882025-05-27T21:15:24.569Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.11817519Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"client":"2","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3892025-05-27T21:15:24.569Z{"msg":"[0]R flush_numbers: [1498, 1498, 1498, 1498, 1498]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.118582113Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"downstairs","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3902025-05-27T21:15:24.569Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.118616674Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"downstairs","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3912025-05-27T21:15:24.569Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.118634415Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"downstairs","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3922025-05-27T21:15:24.569Z{"msg":"[1]R flush_numbers: [1498, 1498, 1498, 1498, 1498]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.118659775Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"downstairs","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3932025-05-27T21:15:24.570Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.118683626Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"downstairs","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3942025-05-27T21:15:24.570Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.118700127Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"downstairs","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3952025-05-27T21:15:24.570Z{"msg":"[2]R flush_numbers: [1497, 1497, 1497, 1497, 1497]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.118724027Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"downstairs","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3962025-05-27T21:15:24.570Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.118740328Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"downstairs","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3972025-05-27T21:15:24.570Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.118764389Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"downstairs","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3982025-05-27T21:15:24.570Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.118779929Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"downstairs","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
3992025-05-27T21:15:24.570Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.11880399Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"downstairs","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4002025-05-27T21:15:24.570Z{"msg":"Next flush: 1499","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.11881927Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"downstairs","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4012025-05-27T21:15:24.570Z{"msg":"extent 0 needs reconciliation: ExtentMetadata { gen: 32, flush: 1498, dirty: false } ExtentMetadata { gen: 32, flush: 1498, dirty: false } ExtentMetadata { gen: 31, flush: 1497, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.118845201Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"mend","":"downstairs","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4022025-05-27T21:15:24.570Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 1498, dirty: false }, ExtentMetadata { gen: 32, flush: 1498, dirty: false }, ExtentMetadata { gen: 31, flush: 1497, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.118864952Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"mend","":"downstairs","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4032025-05-27T21:15:24.570Z{"msg":"found dest for source 0 for extent at index 0 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.118881483Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"mend","":"downstairs","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4042025-05-27T21:15:24.570Z{"msg":"extent 1 needs reconciliation: ExtentMetadata { gen: 32, flush: 1498, dirty: false } ExtentMetadata { gen: 32, flush: 1498, dirty: false } ExtentMetadata { gen: 31, flush: 1497, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.118946365Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"mend","":"downstairs","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4052025-05-27T21:15:24.570Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 1498, dirty: false }, ExtentMetadata { gen: 32, flush: 1498, dirty: false }, ExtentMetadata { gen: 31, flush: 1497, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.118968785Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"mend","":"downstairs","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4062025-05-27T21:15:24.570Z{"msg":"found dest for source 0 for extent at index 1 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.118985566Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"mend","":"downstairs","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4072025-05-27T21:15:24.570Z{"msg":"extent 2 needs reconciliation: ExtentMetadata { gen: 32, flush: 1498, dirty: false } ExtentMetadata { gen: 32, flush: 1498, dirty: false } ExtentMetadata { gen: 31, flush: 1497, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.119003507Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"mend","":"downstairs","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4082025-05-27T21:15:24.570Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 1498, dirty: false }, ExtentMetadata { gen: 32, flush: 1498, dirty: false }, ExtentMetadata { gen: 31, flush: 1497, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.119031577Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"mend","":"downstairs","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4092025-05-27T21:15:24.570Z{"msg":"found dest for source 0 for extent at index 2 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.119050008Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"mend","":"downstairs","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4102025-05-27T21:15:24.570Z{"msg":"extent 3 needs reconciliation: ExtentMetadata { gen: 32, flush: 1498, dirty: false } ExtentMetadata { gen: 32, flush: 1498, dirty: false } ExtentMetadata { gen: 31, flush: 1497, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.119076929Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"mend","":"downstairs","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4112025-05-27T21:15:24.570Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 1498, dirty: false }, ExtentMetadata { gen: 32, flush: 1498, dirty: false }, ExtentMetadata { gen: 31, flush: 1497, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.11910391Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"mend","":"downstairs","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4122025-05-27T21:15:24.570Z{"msg":"found dest for source 0 for extent at index 3 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.11912139Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"mend","":"downstairs","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4132025-05-27T21:15:24.570Z{"msg":"extent 4 needs reconciliation: ExtentMetadata { gen: 32, flush: 1498, dirty: false } ExtentMetadata { gen: 32, flush: 1498, dirty: false } ExtentMetadata { gen: 31, flush: 1497, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.119147831Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"mend","":"downstairs","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4142025-05-27T21:15:24.570Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 1498, dirty: false }, ExtentMetadata { gen: 32, flush: 1498, dirty: false }, ExtentMetadata { gen: 31, flush: 1497, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.119166832Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"mend","":"downstairs","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4152025-05-27T21:15:24.570Z{"msg":"found dest for source 0 for extent at index 4 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.119191473Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"mend","":"downstairs","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4162025-05-27T21:15:24.570Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.119223094Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"client":"0","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4172025-05-27T21:15:24.570Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.119249694Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"client":"1","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4182025-05-27T21:15:24.570Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.119267025Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"client":"2","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4192025-05-27T21:15:24.570Z{"msg":"Full repair list: {ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.119295386Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"downstairs","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4202025-05-27T21:15:24.570Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 1499, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 1499, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 1499, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 1499, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 1499, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.119506352Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"downstairs","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4212025-05-27T21:15:24.570Z{"msg":"starting reconciliation 59c4c14a-894b-4be2-94d6-f1cd869e7562: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.119541894Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"downstairs","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4222025-05-27T21:15:24.570Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.120097912Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"client":"2","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4232025-05-27T21:15:24.578Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.130963171Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"client":"2","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4242025-05-27T21:15:24.582Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.134761253Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"client":"2","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4252025-05-27T21:15:24.585Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.138461632Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"client":"2","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4262025-05-27T21:15:24.589Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.142214793Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"client":"2","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4272025-05-27T21:15:24.592Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.145457007Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"":"downstairs","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4282025-05-27T21:15:24.592Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.145493818Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"client":"0","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4292025-05-27T21:15:24.592Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.145509579Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"client":"1","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4302025-05-27T21:15:24.592Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.145523029Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"client":"2","":"downstairs","session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4312025-05-27T21:15:24.592Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.14554418Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4322025-05-27T21:15:24.592Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.1455567Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4332025-05-27T21:15:24.592Z{"msg":"af9477eb-51d6-42dc-840b-b237a825ab89 is now active with session: 274033b9-91b0-4d8e-8ffd-2a0e1e3871ec","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.145569271Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256,"session_id":"274033b9-91b0-4d8e-8ffd-2a0e1e3871ec"}
4342025-05-27T21:15:24.592Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.145581521Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256}
4352025-05-27T21:15:24.596Z{"msg":"Activated sub_volume af9477eb-51d6-42dc-840b-b237a825ab89","v":0,"name":"crucible","level":30,"time":"2025-05-27T21:15:24.145647763Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256}
4362025-05-27T21:15:24.596ZWait for a query_work_queue command to finish before sending IO
4372025-05-27T21:15:24.596ZDisk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
4382025-05-27T21:15:24.596ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4392025-05-27T21:15:24.596ZRead and Verify all blocks (0..50 range:true)
4402025-05-27T21:15:24.600ZWrote out file "/var/tmp/test_up-build/verify_file"
4412025-05-27T21:15:24.600ZVerify test completed
4422025-05-27T21:15:24.603ZWrote out file "/var/tmp/test_up-build/verify_file"
4432025-05-27T21:15:24.603ZCLIENT: Tests done. All submitted work has been ACK'd
4442025-05-27T21:15:24.603Z----------------------------------------------------------------
4452025-05-27T21:15:24.603Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
4462025-05-27T21:15:24.603ZStates: Active Active Active
4472025-05-27T21:15:24.603ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4482025-05-27T21:15:24.603Z 1000 Acked Read 50 Done Done Done false
4492025-05-27T21:15:24.603Z STATES DS:0 DS:1 DS:2 TOTAL
4502025-05-27T21:15:24.603Z Sent 0 0 0 0
4512025-05-27T21:15:24.603Z Done 1 1 1 3
4522025-05-27T21:15:24.603Z Skipped 0 0 0 0
4532025-05-27T21:15:24.603Z Error 0 0 0 0
4542025-05-27T21:15:24.603ZLast Flush: None None None
4552025-05-27T21:15:24.603ZDownstairs last five completed:
4562025-05-27T21:15:24.603ZUpstairs last five completed: 1000
4572025-05-27T21:15:24.603ZCLIENT: Up:0 ds:1 act:3
4582025-05-27T21:15:28.603Z----------------------------------------------------------------
4592025-05-27T21:15:28.603Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
4602025-05-27T21:15:28.603ZDownstairs last five completed: 1001 1000
4612025-05-27T21:15:28.603ZUpstairs last five completed: 1001 1000
4622025-05-27T21:15:28.603ZCLIENT: Up:0 ds:0 act:3
4632025-05-27T21:15:28.603ZCLIENT: All crucible jobs finished, exiting program
4642025-05-27T21:15:28.606Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-05-27T21:15:28.155871198Z","hostname":"w-01JW9SC4ZQ9YSQWP2B7N5AJD2Z","pid":1256}
4652025-05-27T21:15:28.610ZRepair part 2 passed
4662025-05-27T21:15:28.610Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4672025-05-27T21:15:28.627ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4682025-05-27T21:15:28.627Z 0 000-009 32 32 32 1498 1498 1498 F F F
4692025-05-27T21:15:28.627Z 1 010-019 32 32 32 1498 1498 1498 F F F
4702025-05-27T21:15:28.627Z 2 020-029 32 32 32 1498 1498 1498 F F F
4712025-05-27T21:15:28.627Z 3 030-039 32 32 32 1498 1498 1498 F F F
4722025-05-27T21:15:28.627Z 4 040-049 32 32 32 1498 1498 1498 F F F
4732025-05-27T21:15:28.627ZMax gen: 32, Max flush: 1498
4742025-05-27T21:15:28.630Zdump test passed
4752025-05-27T21:15:28.634Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
4762025-05-27T21:15:28.650Z Extent 1
4772025-05-27T21:15:28.653ZGEN 32 32 32
4782025-05-27T21:15:28.653ZFLUSH_ID 1498 1498 1498
4792025-05-27T21:15:28.653ZDIRTY
4802025-05-27T21:15:28.653Z
4812025-05-27T21:15:28.653ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
4822025-05-27T21:15:28.653Z010 A A A A A A
4832025-05-27T21:15:28.653Z011 A A A A A A
4842025-05-27T21:15:28.653Z012 A A A A A A
4852025-05-27T21:15:28.656Z013 A A A A A A
4862025-05-27T21:15:28.656Z014 A A A A A A
4872025-05-27T21:15:28.656Z015 A A A A A A
4882025-05-27T21:15:28.659Z016 A A A A A A
4892025-05-27T21:15:28.660Z017 A A A A A A
4902025-05-27T21:15:28.663Z018 A A A A A A
4912025-05-27T21:15:28.663Z019 A A A A A A
4922025-05-27T21:15:28.666Zdump extent test passed
4932025-05-27T21:15:28.666Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
4942025-05-27T21:15:28.682ZExtent 2 Block in extent 0 Actual block 20
4952025-05-27T21:15:28.685Z
4962025-05-27T21:15:28.685Z DATA SHA256 VER
4972025-05-27T21:15:28.686Z------ ---------------------------------------------------------------- ---
4982025-05-27T21:15:28.686Z 0 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
4992025-05-27T21:15:28.686Z 1 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
5002025-05-27T21:15:28.686Z 2 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
5012025-05-27T21:15:28.686Z
5022025-05-27T21:15:28.686ZNONCES 0 1 2 DIFF
5032025-05-27T21:15:28.686Z------ ------------------------ ------------------------ ------------------------ -----
5042025-05-27T21:15:28.686Z
5052025-05-27T21:15:28.686Z TAGS 0 1 2 DIFF
5062025-05-27T21:15:28.686Z------ -------------------------------- -------------------------------- -------------------------------- -----
5072025-05-27T21:15:28.686Z
5082025-05-27T21:15:28.686ZHASHES 0 1 2 DIFF
5092025-05-27T21:15:28.686Z------ ---------------- ---------------- ---------------- -----
5102025-05-27T21:15:28.686Z 0 42916ab6fa364c70 42916ab6fa364c70 42916ab6fa364c70
5112025-05-27T21:15:28.686Z
5122025-05-27T21:15:28.689Zdump block test passed
5132025-05-27T21:15:28.689ZInitial upstairs tests have completed, stopping all downstairs
5142025-05-27T21:15:33.757ZCreating 4 larger downstairs regions
5152025-05-27T21:15:33.963ZStarting 4 downstairs
5162025-05-27T21:15:38.970Zdsc restarted at PID: 1269
5172025-05-27T21:15:39.201ZNow do the replace-reconcile test
5182025-05-27T21:15:39.260ZUsing 8840 for the replacement port
5192025-05-27T21:18:49.569ZCompleted test: replace-reconcile
5202025-05-27T21:18:49.573ZNow do the replace-before-active test
5212025-05-27T21:21:47.085ZCompleted test: replace-before-active
5222025-05-27T21:21:47.088ZAll tests have completed, stopping all downstairs
5232025-05-27T21:21:47.143Z
5242025-05-27T21:21:47.143ZAll Tests have passed
5252025-05-27T21:21:47.147Z8:40 Test duration
5262025-05-27T21:21:47.147Z
5272025-05-27T21:21:47.147Zreal 8:40.237591127
5282025-05-27T21:21:47.147Zuser 1:51.894825606
5292025-05-27T21:21:47.147Zsys 13.394217915
5302025-05-27T21:21:47.147Ztrap 0.198209873
5312025-05-27T21:21:47.147Ztflt 0.016985031
5322025-05-27T21:21:47.147Zdflt 0.013435055
5332025-05-27T21:21:47.147Zkflt 0.005434007
5342025-05-27T21:21:47.147Zlock 5:59:57.911230627
5352025-05-27T21:21:47.147Zslp 25:56.743084631
5362025-05-27T21:21:47.147Zlat 16.510632820
5372025-05-27T21:21:47.147Zstop 0.127733602
5382025-05-27T21:21:47.147Z+ echo 'test-up-unencrypted ends'
5392025-05-27T21:21:47.147Ztest-up-unencrypted ends
5402025-05-27T21:21:52.150Zprocess exited: duration 528776 ms, exit code 0
5412025-05-27T21:21:52.150Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
5422025-05-27T21:22:52.194Zexec warning: : stdout descriptor may be held open by a background process; giving up!
5432025-05-27T21:22:52.194Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
5442025-05-27T21:22:52.207Zfound 14 output files
5452025-05-27T21:22:52.207Zuploading: /tmp/test_up-build/dsc-out.txt (41803 bytes)
5462025-05-27T21:22:53.228Zuploaded: /tmp/test_up-build/dsc-out.txt
5472025-05-27T21:22:53.234Zuploading: /tmp/test_up-build/test_up_out.txt (16799214 bytes)
5482025-05-27T21:22:54.566Zuploaded: /tmp/test_up-build/test_up_out.txt
5492025-05-27T21:22:54.571Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (2499 bytes)
5502025-05-27T21:22:55.579Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
5512025-05-27T21:22:55.579Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (548237 bytes)
5522025-05-27T21:22:56.615Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
5532025-05-27T21:22:56.618Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (981853 bytes)
5542025-05-27T21:22:57.662Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
5552025-05-27T21:22:57.666Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (542990 bytes)
5562025-05-27T21:22:58.700Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
5572025-05-27T21:22:58.703Zuploading: /tmp/debug/df.txt (1270 bytes)
5582025-05-27T21:22:59.709Zuploaded: /tmp/debug/df.txt
5592025-05-27T21:22:59.713Zuploading: /tmp/debug/dtrace.txt (478913 bytes)
5602025-05-27T21:23:00.744Zuploaded: /tmp/debug/dtrace.txt
5612025-05-27T21:23:00.747Zuploading: /tmp/debug/iostat.txt (173467 bytes)
5622025-05-27T21:23:00.762Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 173467 -> 175835
5632025-05-27T21:23:01.771Zuploaded: /tmp/debug/iostat.txt
5642025-05-27T21:23:01.774Zuploading: /tmp/debug/mpstat.txt (424876 bytes)
5652025-05-27T21:23:01.800Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 424876 -> 431401
5662025-05-27T21:23:02.809Zuploaded: /tmp/debug/mpstat.txt
5672025-05-27T21:23:02.813Zuploading: /tmp/debug/paging.txt (74368 bytes)
5682025-05-27T21:23:02.821Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 74368 -> 75636
5692025-05-27T21:23:03.827Zuploaded: /tmp/debug/paging.txt
5702025-05-27T21:23:03.830Zuploading: /tmp/debug/prstat.txt (813551 bytes)
5712025-05-27T21:23:03.867Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 813551 -> 827740
5722025-05-27T21:23:04.874Zuploaded: /tmp/debug/prstat.txt
5732025-05-27T21:23:04.878Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
5742025-05-27T21:23:05.884Zuploaded: /tmp/debug/psrinfo.txt
5752025-05-27T21:23:05.888Zuploading: /tmp/debug/upstairs-info.txt (55440 bytes)
5762025-05-27T21:23:06.900Zuploaded: /tmp/debug/upstairs-info.txt