01JMKK62MWKTS15218FZ1Y0Y22: test-up-unencrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01JMKK6ET5VHJT5ZM646M4CQB8

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12025-02-21T06:43:46.118Zjob dependencies complete; ready to run (waiting for 13 m 22 s)
22025-02-21T06:50:14.747Zjob assigned to worker 01JMKM2KSQ08WM8PA2ZPYG4FCX [factory aws, i-0266828b3a2deaffb] (queued for 6 m 28 s)
32025-02-21T06:50:22.506Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42025-02-21T06:50:24.442Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52025-02-21T06:50:24.442Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62025-02-21T06:50:25.479Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72025-02-21T06:50:25.479Zdownloading input: /input/build/work/bins/crutest.gz
82025-02-21T06:50:27.245Zdownloaded input: /input/build/work/bins/crutest.gz
92025-02-21T06:50:27.245Zdownloading input: /input/build/work/bins/dsc.gz
102025-02-21T06:50:27.901Zdownloaded input: /input/build/work/bins/dsc.gz
112025-02-21T06:50:27.904Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122025-02-21T06:50:27.921Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132025-02-21T06:50:27.921Zdownloading input: /input/build/work/scripts/test_ds.sh
142025-02-21T06:50:27.952Zdownloaded input: /input/build/work/scripts/test_ds.sh
152025-02-21T06:50:27.952Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162025-02-21T06:50:27.974Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172025-02-21T06:50:27.974Zdownloading input: /input/build/work/scripts/test_repair.sh
182025-02-21T06:50:27.988Zdownloaded input: /input/build/work/scripts/test_repair.sh
192025-02-21T06:50:27.988Zdownloading input: /input/build/work/scripts/test_replay.sh
202025-02-21T06:50:28.007Zdownloaded input: /input/build/work/scripts/test_replay.sh
212025-02-21T06:50:28.007Zdownloading input: /input/build/work/scripts/test_up.sh
222025-02-21T06:50:28.036Zdownloaded input: /input/build/work/scripts/test_up.sh
232025-02-21T06:50:28.036Zdownloading input: /input/build/work/scripts/upstairs_info.d
242025-02-21T06:50:28.070Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252025-02-21T06:50:28.072Zdownloading input: /input/build/tmp/cargo-test-out.log
262025-02-21T06:50:28.775Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272025-02-21T06:50:28.775Zstarting task 0: "setup"
282025-02-21T06:50:28.783Z++ uname -s
292025-02-21T06:50:28.785Z+ kern=SunOS
302025-02-21T06:50:28.785Z+ build_user=build
312025-02-21T06:50:28.785Z+ build_uid=12345
322025-02-21T06:50:28.785Z+ work_dir=/work
332025-02-21T06:50:28.786Z+ input_dir=/input
342025-02-21T06:50:28.786Z+ [[ 0 == 12345 ]]
352025-02-21T06:50:28.786Z+ case "$kern" in
362025-02-21T06:50:28.786Z+ groupadd -g 12345 build
372025-02-21T06:50:28.788Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382025-02-21T06:50:30.796Z+ zfs create -o mountpoint=/work rpool/work
392025-02-21T06:50:31.108Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402025-02-21T06:50:31.112Z+ home_fs=zfs
412025-02-21T06:50:31.112Z+ [[ zfs == autofs ]]
422025-02-21T06:50:31.112Z+ mkdir -p /home/build
432025-02-21T06:50:31.113Z+ chown build:build /home/build /work
442025-02-21T06:50:33.115Z+ chmod 0700 /home/build /work
452025-02-21T06:50:33.118Zprocess exited: duration 4342 ms, exit code 0
 
462025-02-21T06:50:33.123Zstarting task 1: "authentication"
472025-02-21T06:50:33.141Zprocess exited: duration 17 ms, exit code 0
 
482025-02-21T06:50:33.146Zstarting task 2: "build"
492025-02-21T06:50:33.151Z+ banner cores
502025-02-21T06:50:33.154Z
512025-02-21T06:50:33.154Z #### #### ##### ###### ####
522025-02-21T06:50:33.154Z # # # # # # # #
532025-02-21T06:50:33.154Z # # # # # ##### ####
542025-02-21T06:50:33.154Z # # # ##### # #
552025-02-21T06:50:33.154Z # # # # # # # # #
562025-02-21T06:50:33.154Z #### #### # # ###### ####
572025-02-21T06:50:33.154Z
582025-02-21T06:50:33.157Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
592025-02-21T06:50:33.163Z+ echo 'input bins dir contains:'
602025-02-21T06:50:33.163Zinput bins dir contains:
612025-02-21T06:50:33.166Z+ ls -ltr /input/build/work/bins
622025-02-21T06:50:33.166Ztotal 757032
632025-02-21T06:50:33.166Z-rw-r--r-- 1 root root 111035092 Feb 21 06:50 crucible-downstairs.gz
642025-02-21T06:50:33.166Z-rw-r--r-- 1 root root 101235307 Feb 21 06:50 crucible-hammer.gz
652025-02-21T06:50:33.166Z-rw-r--r-- 1 root root 110141705 Feb 21 06:50 crutest.gz
662025-02-21T06:50:33.166Z-rw-r--r-- 1 root root 64938397 Feb 21 06:50 dsc.gz
672025-02-21T06:50:33.166Z+ banner unpack
682025-02-21T06:50:33.169Z
692025-02-21T06:50:33.169Z # # # # ##### ## #### # #
702025-02-21T06:50:33.169Z # # ## # # # # # # # # #
712025-02-21T06:50:33.169Z # # # # # # # # # # ####
722025-02-21T06:50:33.169Z # # # # # ##### ###### # # #
732025-02-21T06:50:33.169Z # # # ## # # # # # # #
742025-02-21T06:50:33.169Z #### # # # # # #### # #
752025-02-21T06:50:33.169Z
762025-02-21T06:50:33.169Z+ mkdir -p /var/tmp/bins
772025-02-21T06:50:33.171Z+ for t in "$input/bins/"*.gz
782025-02-21T06:50:33.172Z++ basename /input/build/work/bins/crucible-downstairs.gz
792025-02-21T06:50:33.172Z+ b=crucible-downstairs.gz
802025-02-21T06:50:33.174Z+ b=crucible-downstairs
812025-02-21T06:50:33.174Z+ gunzip
822025-02-21T06:50:35.443Z+ chmod +x /var/tmp/bins/crucible-downstairs
832025-02-21T06:50:35.445Z+ for t in "$input/bins/"*.gz
842025-02-21T06:50:35.448Z++ basename /input/build/work/bins/crucible-hammer.gz
852025-02-21T06:50:35.448Z+ b=crucible-hammer.gz
862025-02-21T06:50:35.448Z+ b=crucible-hammer
872025-02-21T06:50:35.448Z+ gunzip
882025-02-21T06:50:37.517Z+ chmod +x /var/tmp/bins/crucible-hammer
892025-02-21T06:50:37.520Z+ for t in "$input/bins/"*.gz
902025-02-21T06:50:37.523Z++ basename /input/build/work/bins/crutest.gz
912025-02-21T06:50:37.523Z+ b=crutest.gz
922025-02-21T06:50:37.523Z+ b=crutest
932025-02-21T06:50:37.523Z+ gunzip
942025-02-21T06:50:39.814Z+ chmod +x /var/tmp/bins/crutest
952025-02-21T06:50:39.817Z+ for t in "$input/bins/"*.gz
962025-02-21T06:50:39.819Z++ basename /input/build/work/bins/dsc.gz
972025-02-21T06:50:39.819Z+ b=dsc.gz
982025-02-21T06:50:39.819Z+ b=dsc
992025-02-21T06:50:39.819Z+ gunzip
1002025-02-21T06:50:41.147Z+ chmod +x /var/tmp/bins/dsc
1012025-02-21T06:50:41.149Z+ export BINDIR=/var/tmp/bins
1022025-02-21T06:50:41.149Z+ BINDIR=/var/tmp/bins
1032025-02-21T06:50:41.150Z+ export RUST_BACKTRACE=1
1042025-02-21T06:50:41.150Z+ RUST_BACKTRACE=1
1052025-02-21T06:50:41.150Z+ jobpid=1145
1062025-02-21T06:50:41.152Z+ echo 'Setup debug logging'
1072025-02-21T06:50:41.152ZSetup debug logging
1082025-02-21T06:50:41.152Z+ mkdir /tmp/debug
1092025-02-21T06:50:41.152Z+ sleep 7200
1102025-02-21T06:50:41.152Z+ psrinfo -v
1112025-02-21T06:50:41.156Z+ df -h
1122025-02-21T06:50:41.163Z+ prstat -d d -mLc 1
1132025-02-21T06:50:41.166Z+ iostat -T d -xn 1
1142025-02-21T06:50:41.166Z+ mpstat -T d 1
1152025-02-21T06:50:41.166Z+ vmstat -T d -p 1
1162025-02-21T06:50:41.168Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1172025-02-21T06:50:41.168Z+ banner test_up_unencrypted
1182025-02-21T06:50:41.168Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1192025-02-21T06:50:41.171Z
1202025-02-21T06:50:41.171Z ##### ###### #### ##### # # ##### # # # #
1212025-02-21T06:50:41.171Z # # # # # # # # # # ## #
1222025-02-21T06:50:41.171Z # ##### #### # # # # # # # # # #
1232025-02-21T06:50:41.171Z # # # # # # ##### # # # # #
1242025-02-21T06:50:41.171Z # # # # # # # # # # # ##
1252025-02-21T06:50:41.171Z # ###### #### # ####### #### # ####### #### # #
1262025-02-21T06:50:41.171Z
1272025-02-21T06:50:41.171Z+ ptime -m bash /input/build/work/scripts/test_up.sh -N unencrypted
1282025-02-21T06:50:41.180Z/input/build/work
1292025-02-21T06:50:41.193ZTurn off color for downstairs dump
1302025-02-21T06:50:41.196Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1312025-02-21T06:50:41.196ZCreating 3 downstairs regions
1322025-02-21T06:50:41.546ZStarting 3 downstairs
1332025-02-21T06:50:41.548Zdsc started at PID: 1190
1342025-02-21T06:50:46.726ZDisable automatic restart on all downstairs
1352025-02-21T06:50:46.778Z
1362025-02-21T06:50:46.779ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1372025-02-21T06:50:46.782ZRunning test: span
1382025-02-21T06:50:50.918ZCompleted test: span
1392025-02-21T06:50:55.925ZRunning test: big
1402025-02-21T06:51:00.072ZCompleted test: big
1412025-02-21T06:51:05.079ZRunning test: dep
1422025-02-21T06:51:17.960ZCompleted test: dep
1432025-02-21T06:51:22.968ZRunning test: balloon
1442025-02-21T06:51:32.654ZCompleted test: balloon
1452025-02-21T06:51:37.661ZRunning test: deactivate
1462025-02-21T06:52:31.873ZCompleted test: deactivate
1472025-02-21T06:52:36.881ZRunning hammer
1482025-02-21T06:52:39.833ZRun repair tests
1492025-02-21T06:52:39.836Z/var/tmp/bins/crutest fill -g 31 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1502025-02-21T06:52:39.896Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.456182115Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"task":"crutest"}
1512025-02-21T06:52:39.899Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.456736482Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"task":"crutest"}
1522025-02-21T06:52:39.899Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.458385942Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"task":"crutest"}
1532025-02-21T06:52:39.899Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.458576577Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227}
1542025-02-21T06:52:39.899Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ae555c374784c668297ef5478b8ff4aaa8d57b4\",\n git_commit_timestamp: \"2025-02-21T06:29:49.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4d91de4e48198da2e33413efdcd9cd2cc0c46688\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.458622004Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227}
1552025-02-21T06:52:39.899Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.458639532Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227}
1562025-02-21T06:52:39.899Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.458657411Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227}
1572025-02-21T06:52:39.902Z{"msg":"Crucible 410d1758-09b3-4984-83d3-138ed32618d9 has session id: 95491ce9-f10b-4aa5-a125-10ca89c63373","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.459219287Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1582025-02-21T06:52:39.902Z{"msg":"Upstairs opts: Upstairs UUID: 410d1758-09b3-4984-83d3-138ed32618d9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.459265903Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1592025-02-21T06:52:39.902Z{"msg":"Crucible stats registered with UUID: 410d1758-09b3-4984-83d3-138ed32618d9","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.45930155Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1602025-02-21T06:52:39.902Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:39.459322969Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1612025-02-21T06:52:39.902Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.46019845Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"task":"crutest"}
1622025-02-21T06:52:39.902Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.460227398Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227}
1632025-02-21T06:52:39.902Z{"msg":"410d1758-09b3-4984-83d3-138ed32618d9 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.4603273Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1642025-02-21T06:52:39.902Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.460371016Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"client":"0","":"downstairs","session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1652025-02-21T06:52:39.902Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.460410193Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"client":"0","":"downstairs","session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1662025-02-21T06:52:39.902Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.46044529Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"client":"1","":"downstairs","session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1672025-02-21T06:52:39.902Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.460478228Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"client":"1","":"downstairs","session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1682025-02-21T06:52:39.902Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.460504466Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"client":"2","":"downstairs","session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1692025-02-21T06:52:39.902Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.460532324Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"client":"2","":"downstairs","session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1702025-02-21T06:52:39.902Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.460559841Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"":"io task","client":"2","":"downstairs","session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1712025-02-21T06:52:39.902Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.460600108Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"":"io task","client":"0","":"downstairs","session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1722025-02-21T06:52:39.902Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.460638415Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"":"io task","client":"1","":"downstairs","session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1732025-02-21T06:52:39.902Z{"msg":"ds_connection connected from Ok(127.0.0.1:64309)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.460664003Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"":"io task","client":"2","":"downstairs","session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1742025-02-21T06:52:39.902Z{"msg":"ds_connection connected from Ok(127.0.0.1:55357)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.460697131Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"":"io task","client":"0","":"downstairs","session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1752025-02-21T06:52:39.902Z{"msg":"ds_connection connected from Ok(127.0.0.1:60390)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.460720049Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"":"io task","client":"1","":"downstairs","session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1762025-02-21T06:52:39.902Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.46122298Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"client":"0","":"downstairs","session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1772025-02-21T06:52:39.902Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.461282795Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"client":"1","":"downstairs","session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1782025-02-21T06:52:39.902Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.461310483Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"client":"2","":"downstairs","session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1792025-02-21T06:52:39.905Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.461833892Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"client":"1","":"downstairs","session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1802025-02-21T06:52:39.906Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.461888347Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"client":"0","":"downstairs","session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1812025-02-21T06:52:39.906Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.462038656Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"client":"2","":"downstairs","session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1822025-02-21T06:52:39.906Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.462334482Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1832025-02-21T06:52:39.906Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.462456153Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1842025-02-21T06:52:39.906Z{"msg":"[0]R flush_numbers: [1495, 1495, 1495, 1495, 1495]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.462501789Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"":"downstairs","":"downstairs","session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1852025-02-21T06:52:39.906Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.462527257Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"":"downstairs","":"downstairs","session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1862025-02-21T06:52:39.906Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.462549685Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"":"downstairs","":"downstairs","session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1872025-02-21T06:52:39.906Z{"msg":"[1]R flush_numbers: [1495, 1495, 1495, 1495, 1495]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.462563904Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"":"downstairs","":"downstairs","session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1882025-02-21T06:52:39.906Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.462577353Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"":"downstairs","":"downstairs","session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1892025-02-21T06:52:39.906Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.462597432Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"":"downstairs","":"downstairs","session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1902025-02-21T06:52:39.906Z{"msg":"[2]R flush_numbers: [1495, 1495, 1495, 1495, 1495]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.462611011Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"":"downstairs","":"downstairs","session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1912025-02-21T06:52:39.906Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.462630519Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"":"downstairs","":"downstairs","session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1922025-02-21T06:52:39.906Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.462643938Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"":"downstairs","":"downstairs","session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1932025-02-21T06:52:39.906ZWait for a query_work_queue command to finish before sending IO
1942025-02-21T06:52:39.906Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.462673975Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"":"downstairs","":"downstairs","session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1952025-02-21T06:52:39.906Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.462709623Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"":"downstairs","":"downstairs","session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1962025-02-21T06:52:39.906Z{"msg":"Next flush: 1496","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.462727811Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"":"downstairs","":"downstairs","session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1972025-02-21T06:52:39.906Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.462760159Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"":"downstairs","":"downstairs","session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1982025-02-21T06:52:39.906Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.462790886Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
1992025-02-21T06:52:39.906Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.462811765Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"client":"0","":"downstairs","session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
2002025-02-21T06:52:39.906Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2012025-02-21T06:52:39.906Z,"time":"2025-02-21T06:52:39.462845602Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"client":"1","":"downstairs","session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
2022025-02-21T06:52:39.906Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Fill test
2032025-02-21T06:52:39.906Z2025-02-21T06:52:39.462880719Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"client":"2","":"downstairs","session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
2042025-02-21T06:52:39.906Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.462898148Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
2052025-02-21T06:52:39.906Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.462918276Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
2062025-02-21T06:52:39.906Z{"msg":"410d1758-09b3-4984-83d3-138ed32618d9 is now active with session: 95491ce9-f10b-4aa5-a125-10ca89c63373","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.462931395Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
2072025-02-21T06:52:39.906Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.462950054Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227,"session_id":"95491ce9-f10b-4aa5-a125-10ca89c63373"}
2082025-02-21T06:52:39.906Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:39.462962443Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227}
2092025-02-21T06:52:39.909ZRead and Verify all blocks (0..50 range:false)
2102025-02-21T06:52:39.915ZWrote out file "/var/tmp/test_up-build/verify_file"
2112025-02-21T06:52:39.915ZCLIENT: Tests done. All submitted work has been ACK'd
2122025-02-21T06:52:39.917Z----------------------------------------------------------------
2132025-02-21T06:52:39.917Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2142025-02-21T06:52:39.917ZStates: Active Active Active
2152025-02-21T06:52:39.917ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2162025-02-21T06:52:39.917Z 1002 Acked Read 50 Done Done Done false
2172025-02-21T06:52:39.917Z STATES DS:0 DS:1 DS:2 TOTAL
2182025-02-21T06:52:39.917Z Sent 0 0 0 0
2192025-02-21T06:52:39.918Z Done 1 1 1 3
2202025-02-21T06:52:39.918Z Skipped 0 0 0 0
2212025-02-21T06:52:39.918Z Error 0 0 0 0
2222025-02-21T06:52:39.918ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2232025-02-21T06:52:39.918ZDownstairs last five completed: 1001 1000
2242025-02-21T06:52:39.918ZUpstairs last five completed: 1002 1001 1000
2252025-02-21T06:52:39.918ZCLIENT: Up:0 ds:1 act:3
2262025-02-21T06:52:43.917Z----------------------------------------------------------------
2272025-02-21T06:52:43.917Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2282025-02-21T06:52:43.917ZDownstairs last five completed: 1003 1002 1001 1000
2292025-02-21T06:52:43.917ZUpstairs last five completed: 1003 1002 1001 1000
2302025-02-21T06:52:43.920ZCLIENT: Up:0 ds:0 act:3
2312025-02-21T06:52:43.920ZCLIENT: All crucible jobs finished, exiting program
2322025-02-21T06:52:43.920Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:43.476389332Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1227}
2332025-02-21T06:52:43.927ZRepair setup passed
2342025-02-21T06:52:43.931ZCopy the region for /var/tmp/test_up-build/8830
2352025-02-21T06:52:43.933Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2362025-02-21T06:52:43.935Z/var/tmp/bins/crutest fill -g 32 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2372025-02-21T06:52:43.993Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.552029075Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"task":"crutest"}
2382025-02-21T06:52:43.995Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.552638547Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"task":"crutest"}
2392025-02-21T06:52:43.995Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.554286748Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"task":"crutest"}
2402025-02-21T06:52:43.995Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.554474163Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233}
2412025-02-21T06:52:43.995Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ae555c374784c668297ef5478b8ff4aaa8d57b4\",\n git_commit_timestamp: \"2025-02-21T06:29:49.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4d91de4e48198da2e33413efdcd9cd2cc0c46688\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.55450998Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233}
2422025-02-21T06:52:43.995Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.554530819Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233}
2432025-02-21T06:52:43.995Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.554545727Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233}
2442025-02-21T06:52:43.998Z{"msg":"Crucible e7a983a6-5260-4acc-8d92-428e2b7aefee has session id: ac08b032-f5d0-472b-8411-723a8bb9cb36","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.555166209Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2452025-02-21T06:52:43.998Z{"msg":"Upstairs opts: Upstairs UUID: e7a983a6-5260-4acc-8d92-428e2b7aefee, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.555239513Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2462025-02-21T06:52:43.998Z{"msg":"Crucible stats registered with UUID: e7a983a6-5260-4acc-8d92-428e2b7aefee","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.55527862Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2472025-02-21T06:52:43.998Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:43.555296388Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2482025-02-21T06:52:43.998Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.55604629Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"task":"crutest"}
2492025-02-21T06:52:43.998Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.556108175Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233}
2502025-02-21T06:52:43.998Z{"msg":"e7a983a6-5260-4acc-8d92-428e2b7aefee active request set","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.556127583Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2512025-02-21T06:52:43.998Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.556149432Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"client":"0","":"downstairs","session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2522025-02-21T06:52:43.998Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.556188618Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"client":"0","":"downstairs","session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2532025-02-21T06:52:43.998Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.556227026Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"client":"1","":"downstairs","session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2542025-02-21T06:52:43.998Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.556276091Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"client":"1","":"downstairs","session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2552025-02-21T06:52:43.998Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.556318488Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"client":"2","":"downstairs","session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2562025-02-21T06:52:43.998Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.556343746Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"client":"2","":"downstairs","session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2572025-02-21T06:52:43.998Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.556381143Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"":"io task","client":"2","":"downstairs","session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2582025-02-21T06:52:43.998Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.556409691Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"":"io task","client":"0","":"downstairs","session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2592025-02-21T06:52:43.998Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.556434239Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"":"io task","client":"1","":"downstairs","session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2602025-02-21T06:52:43.998Z{"msg":"ds_connection connected from Ok(127.0.0.1:40380)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.556458037Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"":"io task","client":"2","":"downstairs","session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2612025-02-21T06:52:43.999Z{"msg":"ds_connection connected from Ok(127.0.0.1:50528)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.556482915Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"":"io task","client":"1","":"downstairs","session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2622025-02-21T06:52:43.999Z{"msg":"ds_connection connected from Ok(127.0.0.1:57318)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.556618545Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"":"io task","client":"0","":"downstairs","session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2632025-02-21T06:52:43.999Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.557131064Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"client":"0","":"downstairs","session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2642025-02-21T06:52:43.999Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.557168441Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"client":"1","":"downstairs","session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2652025-02-21T06:52:43.999Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.55718537Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"client":"2","":"downstairs","session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2662025-02-21T06:52:44.002Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.557770084Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"client":"0","":"downstairs","session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2672025-02-21T06:52:44.002Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.557810411Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"client":"2","":"downstairs","session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2682025-02-21T06:52:44.002Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.55783162Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"client":"1","":"downstairs","session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2692025-02-21T06:52:44.002Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.558192661Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2702025-02-21T06:52:44.002Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.558225789Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2712025-02-21T06:52:44.002Z{"msg":"[0]R flush_numbers: [1496, 1496, 1496, 1496, 1496]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.558247387Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"":"downstairs","":"downstairs","session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2722025-02-21T06:52:44.002Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.558268295Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"":"downstairs","":"downstairs","session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2732025-02-21T06:52:44.002Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.558282154Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"":"downstairs","":"downstairs","session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2742025-02-21T06:52:44.002Z{"msg":"[1]R flush_numbers: [1496, 1496, 1496, 1496, 1496]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.558302023Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"":"downstairs","":"downstairs","session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2752025-02-21T06:52:44.002Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.558315191Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"":"downstairs","":"downstairs","session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2762025-02-21T06:52:44.002Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.55832943Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"":"downstairs","":"downstairs","session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2772025-02-21T06:52:44.002Z{"msg":"[2]R flush_numbers: [1496, 1496, 1496, 1496, 1496]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.558342769Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"":"downstairs","":"downstairs","session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2782025-02-21T06:52:44.002Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.558363238Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"":"downstairs","":"downstairs","session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2792025-02-21T06:52:44.002Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.558376677Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"":"downstairs","":"downstairs","session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2802025-02-21T06:52:44.002Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.558389346Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"":"downstairs","":"downstairs","session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2812025-02-21T06:52:44.002Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.558405655Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"":"downstairs","":"downstairs","session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2822025-02-21T06:52:44.002Z{"msg":"Next flush: 1497","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.558430102Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"":"downstairs","":"downstairs","session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2832025-02-21T06:52:44.002Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.558443581Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"":"downstairs","":"downstairs","session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2842025-02-21T06:52:44.002Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.558456491Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2852025-02-21T06:52:44.002Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.558475479Z","hostname":"Wait for a query_work_queue command to finish before sending IO
2862025-02-21T06:52:44.002Zw-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"client":"0","":"downstairs","session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2872025-02-21T06:52:44.002Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.558496077Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"client":"1","":"downstairs","session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2882025-02-21T06:52:44.002Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.558509606Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"client":"2","":"downstairs","session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2892025-02-21T06:52:44.002Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.558522475Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2902025-02-21T06:52:44.002Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.558541304Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2912025-02-21T06:52:44.002Z{"msg":"e7a983a6-5260-4acc-8d92-428e2b7aefee is now active with session: ac08b032-f5d0-472b-8411-723a8bb9cb36","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.558554273Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2922025-02-21T06:52:44.002Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.558565772Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233,"session_id":"ac08b032-f5d0-472b-8411-723a8bb9cb36"}
2932025-02-21T06:52:44.003Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:43.55858442Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233}
2942025-02-21T06:52:44.003ZDisk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2952025-02-21T06:52:44.003ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
2962025-02-21T06:52:44.003ZFill test
2972025-02-21T06:52:44.005ZRead and Verify all blocks (0..50 range:false)
2982025-02-21T06:52:44.011ZWrote out file "/var/tmp/test_up-build/verify_file"
2992025-02-21T06:52:44.011ZCLIENT: Tests done. All submitted work has been ACK'd
3002025-02-21T06:52:44.013Z----------------------------------------------------------------
3012025-02-21T06:52:44.013Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
3022025-02-21T06:52:44.013ZStates: Active Active Active
3032025-02-21T06:52:44.013ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
3042025-02-21T06:52:44.013Z 1002 Acked Read 50 Done Done Done false
3052025-02-21T06:52:44.013Z STATES DS:0 DS:1 DS:2 TOTAL
3062025-02-21T06:52:44.013Z Sent 0 0 0 0
3072025-02-21T06:52:44.013Z Done 1 1 1 3
3082025-02-21T06:52:44.013Z Skipped 0 0 0 0
3092025-02-21T06:52:44.013Z Error 0 0 0 0
3102025-02-21T06:52:44.014ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
3112025-02-21T06:52:44.014ZDownstairs last five completed: 1001 1000
3122025-02-21T06:52:44.014ZUpstairs last five completed: 1002 1001 1000
3132025-02-21T06:52:44.014ZCLIENT: Up:0 ds:1 act:3
3142025-02-21T06:52:48.013Z----------------------------------------------------------------
3152025-02-21T06:52:48.013Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
3162025-02-21T06:52:48.013ZDownstairs last five completed: 1003 1002 1001 1000
3172025-02-21T06:52:48.013ZUpstairs last five completed: 1003 1002 1001 1000
3182025-02-21T06:52:48.013ZCLIENT: Up:0 ds:0 act:3
3192025-02-21T06:52:48.013ZCLIENT: All crucible jobs finished, exiting program
3202025-02-21T06:52:48.016Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:47.572609363Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1233}
3212025-02-21T06:52:48.020ZRepair part 1 passed
3222025-02-21T06:52:48.022Z
3232025-02-21T06:52:48.022ZKill the current downstairs
3242025-02-21T06:52:48.184ZDownstairs 2 stopped
3252025-02-21T06:52:48.184Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
3262025-02-21T06:52:48.187ZNow put back the original so we have a mismatch
3272025-02-21T06:52:48.187Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
3282025-02-21T06:52:48.189ZRestart downstairs with old directory
3292025-02-21T06:52:48.242Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
3302025-02-21T06:52:48.258ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3312025-02-21T06:52:48.261Z 0 000-009 32 32 31 1497 1497 1496 F F F <---
3322025-02-21T06:52:48.261Z 1 010-019 32 32 31 1497 1497 1496 F F F <---
3332025-02-21T06:52:48.261Z 2 020-029 32 32 31 1497 1497 1496 F F F <---
3342025-02-21T06:52:48.261Z 3 030-039 32 32 31 1497 1497 1496 F F F <---
3352025-02-21T06:52:48.261Z 4 040-049 32 32 31 1497 1497 1496 F F F <---
3362025-02-21T06:52:48.261ZMax gen: 32, Max flush: 1497
3372025-02-21T06:52:48.461ZError: Difference in extent metadata found!
3382025-02-21T06:52:48.461Z
3392025-02-21T06:52:48.461ZStack backtrace:
3402025-02-21T06:52:48.461Z 0: anyhow::error::<impl anyhow::Error>::msg
3412025-02-21T06:52:48.461Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.92/src/backtrace.rs:27:14
3422025-02-21T06:52:48.461Z 1: anyhow::__private::format_err
3432025-02-21T06:52:48.461Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.92/src/lib.rs:691:13
3442025-02-21T06:52:48.478Zdump test found error as expected
3452025-02-21T06:52:48.478Z
3462025-02-21T06:52:48.478Z
3472025-02-21T06:52:48.481Z/var/tmp/bins/crutest verify --range -g 33 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
3482025-02-21T06:52:48.534Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:48.093214748Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"task":"crutest"}
3492025-02-21T06:52:48.537Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:48.093801222Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"task":"crutest"}
3502025-02-21T06:52:48.537Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:48.095447813Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"task":"crutest"}
3512025-02-21T06:52:48.537Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:48.095628319Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241}
3522025-02-21T06:52:48.537Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ae555c374784c668297ef5478b8ff4aaa8d57b4\",\n git_commit_timestamp: \"2025-02-21T06:29:49.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4d91de4e48198da2e33413efdcd9cd2cc0c46688\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:48.095683725Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241}
3532025-02-21T06:52:48.537Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:48.095712012Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241}
3542025-02-21T06:52:48.537Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:48.095730471Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241}
3552025-02-21T06:52:48.540Z{"msg":"Crucible 8ddaf81a-2060-446a-9bf6-d6a73fa65cc9 has session id: 6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:48.096390339Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3562025-02-21T06:52:48.540Z{"msg":"Upstairs opts: Upstairs UUID: 8ddaf81a-2060-446a-9bf6-d6a73fa65cc9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:48.096444085Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3572025-02-21T06:52:48.540Z{"msg":"Crucible stats registered with UUID: 8ddaf81a-2060-446a-9bf6-d6a73fa65cc9","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:48.096478762Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3582025-02-21T06:52:48.540Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:48.09650297Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3592025-02-21T06:52:48.540Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:48.097303678Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"task":"crutest"}
3602025-02-21T06:52:48.541Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:48.097352494Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241}
3612025-02-21T06:52:48.541Z{"msg":"8ddaf81a-2060-446a-9bf6-d6a73fa65cc9 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:48.097375792Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3622025-02-21T06:52:48.541Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:48.097435717Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"client":"0","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3632025-02-21T06:52:48.541Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:48.097510051Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"client":"0","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3642025-02-21T06:52:48.541Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:48.097553338Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"client":"1","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3652025-02-21T06:52:48.541Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:48.097587095Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"client":"1","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3662025-02-21T06:52:48.541Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:48.097628932Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"client":"2","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3672025-02-21T06:52:48.541Z{"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:48.09765974Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"client":"2","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3682025-02-21T06:52:48.541Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:48.097694277Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"io task","client":"2","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3692025-02-21T06:52:48.541Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:48.097728234Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"io task","client":"0","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3702025-02-21T06:52:48.541Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:48.097752212Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"io task","client":"1","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3712025-02-21T06:52:48.541Z{"msg":"ds_connection connected from Ok(127.0.0.1:49609)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:48.09778465Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"io task","client":"0","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3722025-02-21T06:52:48.541Z{"msg":"ds_connection connected from Ok(127.0.0.1:64960)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:48.097818007Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"io task","client":"1","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3732025-02-21T06:52:48.541Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:48.097900281Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"io task","client":"2","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3742025-02-21T06:52:48.541Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:48.097934368Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"io task","client":"2","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3752025-02-21T06:52:48.541Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:48.097959786Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3762025-02-21T06:52:48.541Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:48.097989214Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"io task","client":"2","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3772025-02-21T06:52:48.541Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:48.098230085Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"client":"1","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3782025-02-21T06:52:48.541Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:48.098428559Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"client":"0","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3792025-02-21T06:52:48.541Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:48.09879864Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"client":"1","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3802025-02-21T06:52:48.541Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:48.098896453Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"client":"0","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3812025-02-21T06:52:48.544Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:48.099158312Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3822025-02-21T06:52:48.544Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:48.099240256Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3832025-02-21T06:52:58.542Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.09944261Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"io task","client":"2","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3842025-02-21T06:52:58.542Z{"msg":"ds_connection connected from Ok(127.0.0.1:63657)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.099613966Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"io task","client":"2","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3852025-02-21T06:52:58.546Z{"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.100723239Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"client":"2","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3862025-02-21T06:52:58.546Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.101325592Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"client":"2","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3872025-02-21T06:52:58.546Z{"msg":"[0]R flush_numbers: [1497, 1497, 1497, 1497, 1497]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.101779836Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"downstairs","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3882025-02-21T06:52:58.546Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.101812624Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"downstairs","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3892025-02-21T06:52:58.546Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.101829122Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"downstairs","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3902025-02-21T06:52:58.546Z{"msg":"[1]R flush_numbers: [1497, 1497, 1497, 1497, 1497]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.101843981Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"downstairs","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3912025-02-21T06:52:58.546Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.101864769Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"downstairs","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3922025-02-21T06:52:58.546Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.101879188Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"downstairs","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3932025-02-21T06:52:58.546Z{"msg":"[2]R flush_numbers: [1496, 1496, 1496, 1496, 1496]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.101893227Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"downstairs","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3942025-02-21T06:52:58.546Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.101913025Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"downstairs","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3952025-02-21T06:52:58.546Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.101927555Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"downstairs","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3962025-02-21T06:52:58.546Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.101941413Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"downstairs","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3972025-02-21T06:52:58.546Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.101954282Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"downstairs","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3982025-02-21T06:52:58.546Z{"msg":"Next flush: 1498","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.101975851Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"downstairs","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
3992025-02-21T06:52:58.546Z{"msg":"extent 0 needs reconciliation: ExtentMetadata { gen: 32, flush: 1497, dirty: false } ExtentMetadata { gen: 32, flush: 1497, dirty: false } ExtentMetadata { gen: 31, flush: 1496, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.101992079Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"mend","":"downstairs","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
4002025-02-21T06:52:58.546Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 1497, dirty: false }, ExtentMetadata { gen: 32, flush: 1497, dirty: false }, ExtentMetadata { gen: 31, flush: 1496, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.102014558Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"mend","":"downstairs","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
4012025-02-21T06:52:58.546Z{"msg":"found dest for source 0 for extent at index 0 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.102029047Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"mend","":"downstairs","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
4022025-02-21T06:52:58.546Z{"msg":"extent 1 needs reconciliation: ExtentMetadata { gen: 32, flush: 1497, dirty: false } ExtentMetadata { gen: 32, flush: 1497, dirty: false } ExtentMetadata { gen: 31, flush: 1496, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.102044595Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"mend","":"downstairs","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
4032025-02-21T06:52:58.546Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 1497, dirty: false }, ExtentMetadata { gen: 32, flush: 1497, dirty: false }, ExtentMetadata { gen: 31, flush: 1496, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.102067093Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"mend","":"downstairs","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
4042025-02-21T06:52:58.546Z{"msg":"found dest for source 0 for extent at index 1 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.102081122Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"mend","":"downstairs","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
4052025-02-21T06:52:58.546Z{"msg":"extent 2 needs reconciliation: ExtentMetadata { gen: 32, flush: 1497, dirty: false } ExtentMetadata { gen: 32, flush: 1497, dirty: false } ExtentMetadata { gen: 31, flush: 1496, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.10210599Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"mend","":"downstairs","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
4062025-02-21T06:52:58.546Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 1497, dirty: false }, ExtentMetadata { gen: 32, flush: 1497, dirty: false }, ExtentMetadata { gen: 31, flush: 1496, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.102122079Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"mend","":"downstairs","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
4072025-02-21T06:52:58.546Z{"msg":"found dest for source 0 for extent at index 2 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.102143958Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"mend","":"downstairs","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
4082025-02-21T06:52:58.546Z{"msg":"extent 3 needs reconciliation: ExtentMetadata { gen: 32, flush: 1497, dirty: false } ExtentMetadata { gen: 32, flush: 1497, dirty: false } ExtentMetadata { gen: 31, flush: 1496, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.102159576Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"mend","":"downstairs","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
4092025-02-21T06:52:58.547Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 1497, dirty: false }, ExtentMetadata { gen: 32, flush: 1497, dirty: false }, ExtentMetadata { gen: 31, flush: 1496, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.102175915Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"mend","":"downstairs","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
4102025-02-21T06:52:58.547Z{"msg":"found dest for source 0 for extent at index 3 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.102212562Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"mend","":"downstairs","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
4112025-02-21T06:52:58.547Z{"msg":"extent 4 needs reconciliation: ExtentMetadata { gen: 32, flush: 1497, dirty: false } ExtentMetadata { gen: 32, flush: 1497, dirty: false } ExtentMetadata { gen: 31, flush: 1496, dirty: false }","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.102229261Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"mend","":"downstairs","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
4122025-02-21T06:52:58.547Z{"msg":"extent:0 [ExtentMetadata { gen: 32, flush: 1497, dirty: false }, ExtentMetadata { gen: 32, flush: 1497, dirty: false }, ExtentMetadata { gen: 31, flush: 1496, dirty: false }] => 0","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.10224441Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"mend","":"downstairs","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
4132025-02-21T06:52:58.547Z{"msg":"found dest for source 0 for extent at index 4 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.102258338Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"mend","":"downstairs","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
4142025-02-21T06:52:58.547Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.102280557Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"client":"0","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
4152025-02-21T06:52:58.547Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.102295306Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"client":"1","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
4162025-02-21T06:52:58.547Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.102317524Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"client":"2","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
4172025-02-21T06:52:58.547Z{"msg":"Full repair list: {ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.102334712Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"downstairs","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
4182025-02-21T06:52:58.547Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 1498, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 1498, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 1498, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 1498, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 1498, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.102419816Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"downstairs","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
4192025-02-21T06:52:58.547Z{"msg":"starting reconciliation 3e0edc70-bd1a-4fd0-9254-c005641d417d: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.102436844Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"downstairs","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
4202025-02-21T06:52:58.547Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.102665757Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"client":"2","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
4212025-02-21T06:52:58.555Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.11345049Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"client":"2","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
4222025-02-21T06:52:58.559Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.117032869Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"client":"2","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
4232025-02-21T06:52:58.562Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.120737448Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"client":"2","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
4242025-02-21T06:52:58.566Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.124231294Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"client":"2","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
4252025-02-21T06:52:58.569Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.127231168Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"":"downstairs","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
4262025-02-21T06:52:58.569Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.127269115Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"client":"0","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
4272025-02-21T06:52:58.569Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.127285784Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"client":"1","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
4282025-02-21T06:52:58.572Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.127299703Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"client":"2","":"downstairs","session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
4292025-02-21T06:52:58.572Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.127313602Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
4302025-02-21T06:52:58.572Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.127325851Z","hostname":"Wait for a query_work_queue command to finish before sending IO
4312025-02-21T06:52:58.572Zw-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
4322025-02-21T06:52:58.572Z{"msg":"8ddaf81a-2060-446a-9bf6-d6a73fa65cc9 is now active with session: 6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.12734412Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241,"session_id":"6c7b2aab-36f0-4d36-9ff4-305eaa8a9e63"}
4332025-02-21T06:52:58.572Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:58.127365588Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241}
4342025-02-21T06:52:58.572ZDisk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
4352025-02-21T06:52:58.572ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4362025-02-21T06:52:58.572ZRead and Verify all blocks (0..50 range:true)
4372025-02-21T06:52:58.576ZWrote out file "/var/tmp/test_up-build/verify_file"
4382025-02-21T06:52:58.576ZVerify test completed
4392025-02-21T06:52:58.578ZWrote out file "/var/tmp/test_up-build/verify_file"
4402025-02-21T06:52:58.578ZCLIENT: Tests done. All submitted work has been ACK'd
4412025-02-21T06:52:58.578Z----------------------------------------------------------------
4422025-02-21T06:52:58.578Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
4432025-02-21T06:52:58.578ZStates: Active Active Active
4442025-02-21T06:52:58.578ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4452025-02-21T06:52:58.579Z 1000 Acked Read 50 Done Done Done false
4462025-02-21T06:52:58.579Z STATES DS:0 DS:1 DS:2 TOTAL
4472025-02-21T06:52:58.579Z Sent 0 0 0 0
4482025-02-21T06:52:58.579Z Done 1 1 1 3
4492025-02-21T06:52:58.579Z Skipped 0 0 0 0
4502025-02-21T06:52:58.579Z Error 0 0 0 0
4512025-02-21T06:52:58.579ZLast Flush: None None None
4522025-02-21T06:52:58.579ZDownstairs last five completed:
4532025-02-21T06:52:58.579ZUpstairs last five completed: 1000
4542025-02-21T06:52:58.579ZCLIENT: Up:0 ds:1 act:3
4552025-02-21T06:53:02.578Z----------------------------------------------------------------
4562025-02-21T06:53:02.578Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
4572025-02-21T06:53:02.578ZDownstairs last five completed: 1001 1000
4582025-02-21T06:53:02.578ZUpstairs last five completed: 1001 1000
4592025-02-21T06:53:02.578ZCLIENT: Up:0 ds:0 act:3
4602025-02-21T06:53:02.578ZCLIENT: All crucible jobs finished, exiting program
4612025-02-21T06:53:02.581Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:53:02.135986505Z","hostname":"w-01JMKM2KSQ08WM8PA2ZPYG4FCX","pid":1241}
4622025-02-21T06:53:02.585ZRepair part 2 passed
4632025-02-21T06:53:02.587Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4642025-02-21T06:53:02.603ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4652025-02-21T06:53:02.605Z 0 000-009 32 32 32 1497 1497 1497 F F F
4662025-02-21T06:53:02.605Z 1 010-019 32 32 32 1497 1497 1497 F F F
4672025-02-21T06:53:02.605Z 2 020-029 32 32 32 1497 1497 1497 F F F
4682025-02-21T06:53:02.605Z 3 030-039 32 32 32 1497 1497 1497 F F F
4692025-02-21T06:53:02.605Z 4 040-049 32 32 32 1497 1497 1497 F F F
4702025-02-21T06:53:02.605ZMax gen: 32, Max flush: 1497
4712025-02-21T06:53:02.608Zdump test passed
4722025-02-21T06:53:02.608Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
4732025-02-21T06:53:02.623Z Extent 1
4742025-02-21T06:53:02.623ZGEN 32 32 32
4752025-02-21T06:53:02.623ZFLUSH_ID 1497 1497 1497
4762025-02-21T06:53:02.626ZDIRTY
4772025-02-21T06:53:02.626Z
4782025-02-21T06:53:02.626ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
4792025-02-21T06:53:02.626Z010 A A A A A A
4802025-02-21T06:53:02.629Z011 A A A A A A
4812025-02-21T06:53:02.629Z012 A A A A A A
4822025-02-21T06:53:02.631Z013 A A A A A A
4832025-02-21T06:53:02.631Z014 A A A A A A
4842025-02-21T06:53:02.634Z015 A A A A A A
4852025-02-21T06:53:02.634Z016 A A A A A A
4862025-02-21T06:53:02.636Z017 A A A A A A
4872025-02-21T06:53:02.638Z018 A A A A A A
4882025-02-21T06:53:02.638Z019 A A A A A A
4892025-02-21T06:53:02.641Zdump extent test passed
4902025-02-21T06:53:02.641Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
4912025-02-21T06:53:02.656ZExtent 2 Block in extent 0 Actual block 20
4922025-02-21T06:53:02.656Z
4932025-02-21T06:53:02.659Z DATA SHA256 VER
4942025-02-21T06:53:02.659Z------ ---------------------------------------------------------------- ---
4952025-02-21T06:53:02.662Z 0 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
4962025-02-21T06:53:02.662Z 1 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
4972025-02-21T06:53:02.662Z 2 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
4982025-02-21T06:53:02.662Z
4992025-02-21T06:53:02.662ZNONCES 0 1 2 DIFF
5002025-02-21T06:53:02.662Z------ ------------------------ ------------------------ ------------------------ -----
5012025-02-21T06:53:02.662Z
5022025-02-21T06:53:02.662Z TAGS 0 1 2 DIFF
5032025-02-21T06:53:02.662Z------ -------------------------------- -------------------------------- -------------------------------- -----
5042025-02-21T06:53:02.662Z
5052025-02-21T06:53:02.662ZHASHES 0 1 2 DIFF
5062025-02-21T06:53:02.662Z------ ---------------- ---------------- ---------------- -----
5072025-02-21T06:53:02.662Z 0 42916ab6fa364c70 42916ab6fa364c70 42916ab6fa364c70
5082025-02-21T06:53:02.662Z
5092025-02-21T06:53:02.665Zdump block test passed
5102025-02-21T06:53:02.665ZInitial upstairs tests have completed, stopping all downstairs
5112025-02-21T06:53:07.726ZCreating 4 larger downstairs regions
5122025-02-21T06:53:07.835ZStarting 4 downstairs
5132025-02-21T06:53:12.843Zdsc restarted at PID: 1254
5142025-02-21T06:53:13.068ZNow do the replace-reconcile test
5152025-02-21T06:53:13.124ZUsing 8840 for the replacement port
5162025-02-21T06:55:54.472ZCompleted test: replace-reconcile
5172025-02-21T06:55:54.474ZNow do the replace-before-active test
5182025-02-21T06:58:38.943ZCompleted test: replace-before-active
5192025-02-21T06:58:38.946ZAll tests have completed, stopping all downstairs
5202025-02-21T06:58:39.002Z
5212025-02-21T06:58:39.002ZAll Tests have passed
5222025-02-21T06:58:39.005Z7:58 Test duration
5232025-02-21T06:58:39.005Z
5242025-02-21T06:58:39.005Zreal 7:57.781636799
5252025-02-21T06:58:39.005Zuser 1:03.646038387
5262025-02-21T06:58:39.005Zsys 9.996642932
5272025-02-21T06:58:39.005Ztrap 0.079377813
5282025-02-21T06:58:39.005Ztflt 0.018372578
5292025-02-21T06:58:39.005Zdflt 0.012006063
5302025-02-21T06:58:39.005Zkflt 0.000971280
5312025-02-21T06:58:39.005Zlock 5:45:30.244015841
5322025-02-21T06:58:39.005Zslp 24:48.102686742
5332025-02-21T06:58:39.005Zlat 7.536982056
5342025-02-21T06:58:39.005Zstop 0.123920294
5352025-02-21T06:58:39.005Z+ echo 'test-up-unencrypted ends'
5362025-02-21T06:58:39.005Ztest-up-unencrypted ends
5372025-02-21T06:58:44.009Zprocess exited: duration 485808 ms, exit code 0
5382025-02-21T06:58:44.009Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
5392025-02-21T06:59:44.057Zexec warning: : stdout descriptor may be held open by a background process; giving up!
5402025-02-21T06:59:44.057Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
5412025-02-21T06:59:44.068Zfound 14 output files
5422025-02-21T06:59:44.068Zuploading: /tmp/test_up-build/dsc-out.txt (37469 bytes)
5432025-02-21T06:59:45.081Zuploaded: /tmp/test_up-build/dsc-out.txt
5442025-02-21T06:59:45.081Zuploading: /tmp/test_up-build/test_up_out.txt (7762358 bytes)
5452025-02-21T06:59:46.164Zuploaded: /tmp/test_up-build/test_up_out.txt
5462025-02-21T06:59:46.166Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (253603 bytes)
5472025-02-21T06:59:47.179Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
5482025-02-21T06:59:47.179Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (144655 bytes)
5492025-02-21T06:59:48.191Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
5502025-02-21T06:59:48.194Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (250996 bytes)
5512025-02-21T06:59:49.206Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
5522025-02-21T06:59:49.209Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (139406 bytes)
5532025-02-21T06:59:50.219Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
5542025-02-21T06:59:50.221Zuploading: /tmp/debug/df.txt (1270 bytes)
5552025-02-21T06:59:51.230Zuploaded: /tmp/debug/df.txt
5562025-02-21T06:59:51.230Zuploading: /tmp/debug/dtrace.txt (459203 bytes)
5572025-02-21T06:59:52.246Zuploaded: /tmp/debug/dtrace.txt
5582025-02-21T06:59:52.246Zuploading: /tmp/debug/iostat.txt (163453 bytes)
5592025-02-21T06:59:52.255Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 163453 -> 166162
5602025-02-21T06:59:53.261Zuploaded: /tmp/debug/iostat.txt
5612025-02-21T06:59:53.261Zuploading: /tmp/debug/mpstat.txt (396396 bytes)
5622025-02-21T06:59:53.272Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 396396 -> 403696
5632025-02-21T06:59:54.279Zuploaded: /tmp/debug/mpstat.txt
5642025-02-21T06:59:54.279Zuploading: /tmp/debug/paging.txt (71678 bytes)
5652025-02-21T06:59:54.285Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 71678 -> 73112
5662025-02-21T06:59:55.291Zuploaded: /tmp/debug/paging.txt
5672025-02-21T06:59:55.291Zuploading: /tmp/debug/prstat.txt (744494 bytes)
5682025-02-21T06:59:55.306Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 744494 -> 760054
5692025-02-21T06:59:56.312Zuploaded: /tmp/debug/prstat.txt
5702025-02-21T06:59:56.312Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
5712025-02-21T06:59:57.320Zuploaded: /tmp/debug/psrinfo.txt
5722025-02-21T06:59:57.320Zuploading: /tmp/debug/upstairs-info.txt (50600 bytes)
5732025-02-21T06:59:58.331Zuploaded: /tmp/debug/upstairs-info.txt