01JGNKQ7R0NJ8B1MFXEWWP065Q: test-up-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01JGNKQGFNAY53AYH36P4GEP1Q

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12025-01-03T08:29:19.941Zjob dependencies complete; ready to run (waiting for 13 m 55 s)
22025-01-03T08:30:30.120Zjob assigned to worker 01JGNMH822F3VW2FFS29KX8E6C [factory aws, i-040a387fc682d5336] (queued for 1 m 10 s)
32025-01-03T08:30:39.640Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42025-01-03T08:30:40.760Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52025-01-03T08:30:40.760Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62025-01-03T08:30:41.811Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72025-01-03T08:30:41.811Zdownloading input: /input/build/work/bins/crutest.gz
82025-01-03T08:30:42.911Zdownloaded input: /input/build/work/bins/crutest.gz
92025-01-03T08:30:42.911Zdownloading input: /input/build/work/bins/dsc.gz
102025-01-03T08:30:43.581Zdownloaded input: /input/build/work/bins/dsc.gz
112025-01-03T08:30:43.583Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122025-01-03T08:30:43.596Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132025-01-03T08:30:43.596Zdownloading input: /input/build/work/scripts/test_ds.sh
142025-01-03T08:30:43.612Zdownloaded input: /input/build/work/scripts/test_ds.sh
152025-01-03T08:30:43.612Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162025-01-03T08:30:43.631Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172025-01-03T08:30:43.631Zdownloading input: /input/build/work/scripts/test_repair.sh
182025-01-03T08:30:43.649Zdownloaded input: /input/build/work/scripts/test_repair.sh
192025-01-03T08:30:43.649Zdownloading input: /input/build/work/scripts/test_replay.sh
202025-01-03T08:30:43.668Zdownloaded input: /input/build/work/scripts/test_replay.sh
212025-01-03T08:30:43.668Zdownloading input: /input/build/work/scripts/test_up.sh
222025-01-03T08:30:43.705Zdownloaded input: /input/build/work/scripts/test_up.sh
232025-01-03T08:30:43.705Zdownloading input: /input/build/work/scripts/upstairs_info.d
242025-01-03T08:30:43.724Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252025-01-03T08:30:43.726Zdownloading input: /input/build/tmp/cargo-test-out.log
262025-01-03T08:30:44.442Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272025-01-03T08:30:44.443Zstarting task 0: "setup"
282025-01-03T08:30:44.453Z++ uname -s
292025-01-03T08:30:44.455Z+ kern=SunOS
302025-01-03T08:30:44.455Z+ case "$kern" in
312025-01-03T08:30:44.455Z+ groupadd -g 12345 build
322025-01-03T08:30:44.458Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332025-01-03T08:30:46.464Z+ zfs create -o mountpoint=/work rpool/work
342025-01-03T08:30:46.621Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352025-01-03T08:30:46.623Z+ home_fs=zfs
362025-01-03T08:30:46.623Z+ [[ zfs == autofs ]]
372025-01-03T08:30:46.623Z+ mkdir -p /home/build
382025-01-03T08:30:46.626Z+ chown build:build /home/build /work
392025-01-03T08:30:48.627Z+ chmod 0700 /home/build /work
402025-01-03T08:30:48.630Zprocess exited: duration 4187 ms, exit code 0
 
412025-01-03T08:30:48.635Zstarting task 1: "authentication"
422025-01-03T08:30:48.652Zprocess exited: duration 16 ms, exit code 0
 
432025-01-03T08:30:48.657Zstarting task 2: "build"
442025-01-03T08:30:48.662Z+ banner cores
452025-01-03T08:30:48.684Z
462025-01-03T08:30:48.684Z #### #### ##### ###### ####
472025-01-03T08:30:48.684Z # # # # # # # #
482025-01-03T08:30:48.684Z # # # # # ##### ####
492025-01-03T08:30:48.684Z # # # ##### # #
502025-01-03T08:30:48.684Z # # # # # # # # #
512025-01-03T08:30:48.684Z #### #### # # ###### ####
522025-01-03T08:30:48.684Z
532025-01-03T08:30:48.686Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542025-01-03T08:30:48.692Z+ echo 'input bins dir contains:'
552025-01-03T08:30:48.692Zinput bins dir contains:
562025-01-03T08:30:48.694Z+ ls -ltr /input/build/work/bins
572025-01-03T08:30:48.694Ztotal 761586
582025-01-03T08:30:48.694Z-rw-r--r-- 1 root root 109299420 Jan 3 08:30 crucible-downstairs.gz
592025-01-03T08:30:48.694Z-rw-r--r-- 1 root root 104596677 Jan 3 08:30 crucible-hammer.gz
602025-01-03T08:30:48.694Z-rw-r--r-- 1 root root 109233638 Jan 3 08:30 crutest.gz
612025-01-03T08:30:48.695Z-rw-r--r-- 1 root root 66553821 Jan 3 08:30 dsc.gz
622025-01-03T08:30:48.697Z+ banner unpack
632025-01-03T08:30:48.697Z
642025-01-03T08:30:48.697Z # # # # ##### ## #### # #
652025-01-03T08:30:48.697Z # # ## # # # # # # # # #
662025-01-03T08:30:48.698Z # # # # # # # # # # ####
672025-01-03T08:30:48.698Z # # # # # ##### ###### # # #
682025-01-03T08:30:48.698Z # # # ## # # # # # # #
692025-01-03T08:30:48.698Z #### # # # # # #### # #
702025-01-03T08:30:48.698Z
712025-01-03T08:30:48.698Z+ mkdir -p /var/tmp/bins
722025-01-03T08:30:48.700Z+ for t in "$input/bins/"*.gz
732025-01-03T08:30:48.700Z++ basename /input/build/work/bins/crucible-downstairs.gz
742025-01-03T08:30:48.703Z+ b=crucible-downstairs.gz
752025-01-03T08:30:48.703Z+ b=crucible-downstairs
762025-01-03T08:30:48.703Z+ gunzip
772025-01-03T08:30:50.959Z+ chmod +x /var/tmp/bins/crucible-downstairs
782025-01-03T08:30:50.961Z+ for t in "$input/bins/"*.gz
792025-01-03T08:30:50.963Z++ basename /input/build/work/bins/crucible-hammer.gz
802025-01-03T08:30:50.963Z+ b=crucible-hammer.gz
812025-01-03T08:30:50.963Z+ b=crucible-hammer
822025-01-03T08:30:50.963Z+ gunzip
832025-01-03T08:30:53.108Z+ chmod +x /var/tmp/bins/crucible-hammer
842025-01-03T08:30:53.111Z+ for t in "$input/bins/"*.gz
852025-01-03T08:30:53.113Z++ basename /input/build/work/bins/crutest.gz
862025-01-03T08:30:53.113Z+ b=crutest.gz
872025-01-03T08:30:53.113Z+ b=crutest
882025-01-03T08:30:53.113Z+ gunzip
892025-01-03T08:30:55.417Z+ chmod +x /var/tmp/bins/crutest
902025-01-03T08:30:55.420Z+ for t in "$input/bins/"*.gz
912025-01-03T08:30:55.422Z++ basename /input/build/work/bins/dsc.gz
922025-01-03T08:30:55.422Z+ b=dsc.gz
932025-01-03T08:30:55.422Z+ b=dsc
942025-01-03T08:30:55.422Z+ gunzip
952025-01-03T08:30:56.807Z+ chmod +x /var/tmp/bins/dsc
962025-01-03T08:30:56.810Z+ export BINDIR=/var/tmp/bins
972025-01-03T08:30:56.810Z+ BINDIR=/var/tmp/bins
982025-01-03T08:30:56.810Z+ jobpid=1139
992025-01-03T08:30:56.810Z+ echo 'Setup debug logging'
1002025-01-03T08:30:56.810ZSetup debug logging
1012025-01-03T08:30:56.810Z+ mkdir /tmp/debug
1022025-01-03T08:30:56.810Z+ sleep 3600
1032025-01-03T08:30:56.814Z+ psrinfo -v
1042025-01-03T08:30:56.816Z+ df -h
1052025-01-03T08:30:56.840Z+ prstat -d d -mLc 1
1062025-01-03T08:30:56.843Z+ iostat -T d -xn 1
1072025-01-03T08:30:56.843Z+ mpstat -T d 1
1082025-01-03T08:30:56.843Z+ vmstat -T d -p 1
1092025-01-03T08:30:56.845Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1102025-01-03T08:30:56.845Z+ banner test_up_encrypted
1112025-01-03T08:30:56.845Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1122025-01-03T08:30:56.848Z
1132025-01-03T08:30:56.848Z ##### ###### #### ##### # # ##### ###### # #
1142025-01-03T08:30:56.848Z # # # # # # # # # ## #
1152025-01-03T08:30:56.848Z # ##### #### # # # # # ##### # # #
1162025-01-03T08:30:56.849Z # # # # # # ##### # # # #
1172025-01-03T08:30:56.849Z # # # # # # # # # # ##
1182025-01-03T08:30:56.849Z # ###### #### # ####### #### # ####### ###### # #
1192025-01-03T08:30:56.849Z
1202025-01-03T08:30:56.849Z+ ptime -m bash /input/build/work/scripts/test_up.sh -N encrypted
1212025-01-03T08:30:56.856Z/input/build/work
1222025-01-03T08:30:56.888ZTurn off color for downstairs dump
1232025-01-03T08:30:56.920ZUpstairs using key: 19Vu0KYX6uqE7olrTYPbVXU+Lq93I99hvQcKpwGNpMM=
1242025-01-03T08:30:56.923Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1252025-01-03T08:30:56.924ZCreating 3 downstairs regions
1262025-01-03T08:30:57.241ZStarting 3 downstairs
1272025-01-03T08:30:57.244Zdsc started at PID: 1185
1282025-01-03T08:31:02.421ZDisable automatic restart on all downstairs
1292025-01-03T08:31:02.477Z
1302025-01-03T08:31:02.477ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1312025-01-03T08:31:02.481ZRunning test: span
1322025-01-03T08:31:06.625ZCompleted test: span
1332025-01-03T08:31:11.632ZRunning test: big
1342025-01-03T08:31:15.988ZCompleted test: big
1352025-01-03T08:31:20.995ZRunning test: dep
1362025-01-03T08:32:13.441ZCompleted test: dep
1372025-01-03T08:32:18.448ZRunning test: balloon
1382025-01-03T08:32:59.495ZCompleted test: balloon
1392025-01-03T08:33:04.503ZRunning test: deactivate
1402025-01-03T08:33:59.200ZCompleted test: deactivate
1412025-01-03T08:34:04.206ZRunning hammer
1422025-01-03T08:34:17.098ZRun repair tests
1432025-01-03T08:34:17.101Z/var/tmp/bins/crutest fill -g 31 -q --key 19Vu0KYX6uqE7olrTYPbVXU+Lq93I99hvQcKpwGNpMM= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1442025-01-03T08:34:17.161Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.257902027Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"task":"crutest"}
1452025-01-03T08:34:17.164Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.2584605Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"task":"crutest"}
1462025-01-03T08:34:17.164Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.26019456Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"task":"crutest"}
1472025-01-03T08:34:17.164Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.260407545Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222}
1482025-01-03T08:34:17.164Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.260454026Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222}
1492025-01-03T08:34:17.164Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.260480777Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222}
1502025-01-03T08:34:17.164Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.260500027Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222}
1512025-01-03T08:34:17.166Z{"msg":"Crucible b5de79f1-eb57-4771-baf1-17b288c7f931 has session id: 27b8a0f6-800d-412e-8e67-b9e0fe27bb14","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.261017569Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1522025-01-03T08:34:17.166Z{"msg":"Upstairs opts: Upstairs UUID: b5de79f1-eb57-4771-baf1-17b288c7f931, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.261078021Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1532025-01-03T08:34:17.166Z{"msg":"Crucible stats registered with UUID: b5de79f1-eb57-4771-baf1-17b288c7f931","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.261097991Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1542025-01-03T08:34:17.198Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.295065734Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"task":"crutest"}
1552025-01-03T08:34:17.200Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.295125946Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222}
1562025-01-03T08:34:17.200Z{"msg":"b5de79f1-eb57-4771-baf1-17b288c7f931 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.295170867Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1572025-01-03T08:34:17.200Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.295261629Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"client":"0","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1582025-01-03T08:34:17.200Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.295346811Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"client":"0","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1592025-01-03T08:34:17.200Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.295386132Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"client":"1","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1602025-01-03T08:34:17.200Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.295436273Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"client":"1","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1612025-01-03T08:34:17.200Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.295472523Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"client":"2","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1622025-01-03T08:34:17.200Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.295507314Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"client":"2","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1632025-01-03T08:34:17.201Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.295537955Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"":"io task","client":"2","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1642025-01-03T08:34:17.201Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.295571026Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1652025-01-03T08:34:17.201Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.295605766Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"":"io task","client":"1","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1662025-01-03T08:34:17.201Z{"msg":"ds_connection connected from Ok(127.0.0.1:35250)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.295691839Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1672025-01-03T08:34:17.201Z{"msg":"ds_connection connected from Ok(127.0.0.1:50817)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.29573771Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"":"io task","client":"2","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1682025-01-03T08:34:17.201Z{"msg":"ds_connection connected from Ok(127.0.0.1:46053)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.29577479Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"":"io task","client":"1","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1692025-01-03T08:34:17.201Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.296252401Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"client":"2","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1702025-01-03T08:34:17.201Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.296294833Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"client":"2","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1712025-01-03T08:34:17.201Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.296310503Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"client":"1","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1722025-01-03T08:34:17.201Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.296324393Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"client":"1","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1732025-01-03T08:34:17.201Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.296357244Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"client":"0","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1742025-01-03T08:34:17.201Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.296372774Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"client":"0","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1752025-01-03T08:34:17.201Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.296950648Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"client":"2","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1762025-01-03T08:34:17.201Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.296993289Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"client":"0","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1772025-01-03T08:34:17.201Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.297024939Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"client":"1","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1782025-01-03T08:34:17.205Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.297403028Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"client":"1","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1792025-01-03T08:34:17.205Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.297440509Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"client":"1","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1802025-01-03T08:34:17.205Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.297456309Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1812025-01-03T08:34:17.205Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.29747596Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"client":"2","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1822025-01-03T08:34:17.205Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.29750002Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"client":"2","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1832025-01-03T08:34:17.205Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.297514101Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1842025-01-03T08:34:17.205Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.297541931Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"client":"0","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1852025-01-03T08:34:17.205Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.297569992Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"client":"0","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1862025-01-03T08:34:17.205Z{"msg":"[0]R flush_numbers: [1497, 1497, 1497, 1497, 1497]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.297596603Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"":"downstairs","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1872025-01-03T08:34:17.205Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.297618003Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"":"downstairs","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1882025-01-03T08:34:17.205Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
1892025-01-03T08:34:17.205Z,"time":"2025-01-03T08:34:16.297649574Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"":"downstairs","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1902025-01-03T08:34:17.205Z{"msg":"[1]R flush_numbers: [1497, 1497, 1497, 1497, 1497]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.297691435Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"":"downstairs","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1912025-01-03T08:34:17.205Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.297720215Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"":"downstairs","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1922025-01-03T08:34:17.205Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.297734846Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"":"downstairs","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1932025-01-03T08:34:17.205Z{"msg":"[2]R flush_numbers: [1497, 1497, 1497, 1497, 1497]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.297762616Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"":"downstairs","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1942025-01-03T08:34:17.205Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.297801997Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"":"downstairs","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1952025-01-03T08:34:17.205Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.297834268Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"":"downstairs","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1962025-01-03T08:34:17.205Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.297855849Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"":"downstairs","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1972025-01-03T08:34:17.205Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.297892529Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"":"downstairs","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1982025-01-03T08:34:17.205Z{"msg":"Next flush: 1498","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.29792564Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"":"downstairs","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
1992025-01-03T08:34:17.205Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.29794392Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"":"downstairs","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
2002025-01-03T08:34:17.205ZDisk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2012025-01-03T08:34:17.205Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.297980031Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
2022025-01-03T08:34:17.205Z{"msg":"Transition from WaitQuorum to Active"Fill test
2032025-01-03T08:34:17.205Z,"v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.298015432Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"client":"0","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
2042025-01-03T08:34:17.205Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.298049373Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"client":"1","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
2052025-01-03T08:34:17.205Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.298072703Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"client":"2","":"downstairs","session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
2062025-01-03T08:34:17.205Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.298095784Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
2072025-01-03T08:34:17.205Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.298126775Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
2082025-01-03T08:34:17.205Z{"msg":"b5de79f1-eb57-4771-baf1-17b288c7f931 is now active with session: 27b8a0f6-800d-412e-8e67-b9e0fe27bb14","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.298165725Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
2092025-01-03T08:34:17.206Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.298198996Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222,"session_id":"27b8a0f6-800d-412e-8e67-b9e0fe27bb14"}
2102025-01-03T08:34:17.206Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:16.298233257Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222}
2112025-01-03T08:34:17.240ZRead and Verify all blocks (0..50 range:false)
2122025-01-03T08:34:17.278ZWrote out file "/var/tmp/test_up-build/verify_file"
2132025-01-03T08:34:17.279ZCLIENT: Tests done. All submitted work has been ACK'd
2142025-01-03T08:34:17.281Z----------------------------------------------------------------
2152025-01-03T08:34:17.281Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2162025-01-03T08:34:17.281ZStates: Active Active Active
2172025-01-03T08:34:17.281ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2182025-01-03T08:34:17.281Z 1002 Acked Read 50 Done Sent Sent false
2192025-01-03T08:34:17.281Z STATES DS:0 DS:1 DS:2 TOTAL
2202025-01-03T08:34:17.281Z Sent 0 1 1 2
2212025-01-03T08:34:17.281Z Done 1 0 0 1
2222025-01-03T08:34:17.281Z Skipped 0 0 0 0
2232025-01-03T08:34:17.281Z Error 0 0 0 0
2242025-01-03T08:34:17.281ZLast Flush: 1001 1001 1001
2252025-01-03T08:34:17.281ZDownstairs last five completed: 1001 1000
2262025-01-03T08:34:17.281ZUpstairs last five completed: 1002 1001 1000
2272025-01-03T08:34:17.281ZCLIENT: Up:0 ds:1 act:3
2282025-01-03T08:34:21.281Z----------------------------------------------------------------
2292025-01-03T08:34:21.281Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2302025-01-03T08:34:21.283ZDownstairs last five completed: 1003 1002 1001 1000
2312025-01-03T08:34:21.283ZUpstairs last five completed: 1003 1002 1001 1000
2322025-01-03T08:34:21.283ZCLIENT: Up:0 ds:0 act:3
2332025-01-03T08:34:21.283ZCLIENT: All crucible jobs finished, exiting program
2342025-01-03T08:34:21.285Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:34:20.378383275Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1222}
2352025-01-03T08:34:21.293ZRepair setup passed
2362025-01-03T08:34:21.297ZCopy the region for /var/tmp/test_up-build/8830
2372025-01-03T08:34:21.299Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2382025-01-03T08:34:21.301Z/var/tmp/bins/crutest fill -g 32 -q --key 19Vu0KYX6uqE7olrTYPbVXU+Lq93I99hvQcKpwGNpMM= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2392025-01-03T08:34:21.361Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.457651042Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"task":"crutest"}
2402025-01-03T08:34:21.363Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.458252176Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"task":"crutest"}
2412025-01-03T08:34:21.363Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.459962546Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"task":"crutest"}
2422025-01-03T08:34:21.363Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.460171361Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228}
2432025-01-03T08:34:21.363Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.460211472Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228}
2442025-01-03T08:34:21.363Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.460226302Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228}
2452025-01-03T08:34:21.364Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.460248442Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228}
2462025-01-03T08:34:21.366Z{"msg":"Crucible e120f9e0-dbff-4667-854d-a272a5354f77 has session id: 18674125-3bbe-49cd-9b34-e43c80f24062","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.460772434Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2472025-01-03T08:34:21.366Z{"msg":"Upstairs opts: Upstairs UUID: e120f9e0-dbff-4667-854d-a272a5354f77, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.460836586Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2482025-01-03T08:34:21.366Z{"msg":"Crucible stats registered with UUID: e120f9e0-dbff-4667-854d-a272a5354f77","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.460873667Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2492025-01-03T08:34:21.398Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.495176978Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"task":"crutest"}
2502025-01-03T08:34:21.398Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.495237549Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228}
2512025-01-03T08:34:21.398Z{"msg":"e120f9e0-dbff-4667-854d-a272a5354f77 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.49526517Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2522025-01-03T08:34:21.401Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.495341442Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"client":"0","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2532025-01-03T08:34:21.401Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.495395563Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"client":"0","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2542025-01-03T08:34:21.401Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.495433004Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"client":"1","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2552025-01-03T08:34:21.401Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.495494425Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"client":"1","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2562025-01-03T08:34:21.401Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.495548046Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"client":"2","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2572025-01-03T08:34:21.401Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.495595677Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"client":"2","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2582025-01-03T08:34:21.401Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.495636548Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"":"io task","client":"2","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2592025-01-03T08:34:21.401Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.495750631Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"":"io task","client":"0","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2602025-01-03T08:34:21.401Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.495812662Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"":"io task","client":"1","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2612025-01-03T08:34:21.401Z{"msg":"ds_connection connected from Ok(127.0.0.1:40806)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.495851823Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"":"io task","client":"0","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2622025-01-03T08:34:21.401Z{"msg":"ds_connection connected from Ok(127.0.0.1:40779)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.495899924Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"":"io task","client":"1","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2632025-01-03T08:34:21.401Z{"msg":"ds_connection connected from Ok(127.0.0.1:49779)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.495951046Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"":"io task","client":"2","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2642025-01-03T08:34:21.401Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.496485138Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"client":"0","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2652025-01-03T08:34:21.401Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.496524299Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"client":"0","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2662025-01-03T08:34:21.401Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.496547329Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"client":"1","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2672025-01-03T08:34:21.401Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.4965742Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"client":"1","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2682025-01-03T08:34:21.401Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.496611711Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"client":"2","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2692025-01-03T08:34:21.401Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.496634011Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"client":"2","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2702025-01-03T08:34:21.401Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.497252816Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"client":"0","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2712025-01-03T08:34:21.401Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.49743631Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"client":"1","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2722025-01-03T08:34:21.401Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.497510001Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"client":"2","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2732025-01-03T08:34:21.401Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.497647985Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"client":"0","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2742025-01-03T08:34:21.401Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.497685545Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"client":"0","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2752025-01-03T08:34:21.401Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.497701446Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2762025-01-03T08:34:21.404Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.497791648Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"client":"2","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2772025-01-03T08:34:21.404Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.497819489Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"client":"2","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2782025-01-03T08:34:21.404Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.497833979Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2792025-01-03T08:34:21.405Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.49786417Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"client":"1","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2802025-01-03T08:34:21.405Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.49787857Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"client":"1","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2812025-01-03T08:34:21.405Z{"msg":"[0]R flush_numbers: [1498, 1498, 1498, 1498, 1498]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.497899781Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"":"downstairs","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2822025-01-03T08:34:21.405Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.497914211Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"":"downstairs","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2832025-01-03T08:34:21.405Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.497927781Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"":"downstairs","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2842025-01-03T08:34:21.405Z{"msg":"[1]R flush_numbers: [1498, 1498, 1498, 1498, 1498]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.497948162Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"":"downstairs","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2852025-01-03T08:34:21.405Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.497961542Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"":"downstairs","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2862025-01-03T08:34:21.405Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.497974942Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"":"downstairs","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2872025-01-03T08:34:21.405Z{"msg":"[2]R flush_numbers: [1498, 1498, 1498, 1498, 1498]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.498008853Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"":"downstairs","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2882025-01-03T08:34:21.405Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.498023923Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"":"downstairs","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2892025-01-03T08:34:21.405Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.498037124Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"":"downstairs","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2902025-01-03T08:34:21.405Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.498051854Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"":"downstairs","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2912025-01-03T08:34:21.405Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.498065354Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"":"downstairs","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2922025-01-03T08:34:21.405Z{"msg":"Next flush: 1499","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.498079095Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"":"downstairs","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2932025-01-03T08:34:21.405Z{"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
2942025-01-03T08:34:21.405Z,"time":"2025-01-03T08:34:20.498092035Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"":"downstairs","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2952025-01-03T08:34:21.405Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.498111195Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2962025-01-03T08:34:21.405Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.498123326Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"client":"0","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2972025-01-03T08:34:21.405Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.498144576Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"client":"1","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2982025-01-03T08:34:21.405Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.498159307Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"client":"2","":"downstairs","session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
2992025-01-03T08:34:21.405Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.498172967Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
3002025-01-03T08:34:21.405Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.498192707Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
3012025-01-03T08:34:21.405Z{"msg":"e120f9e0-dbff-4667-854d-a272a5354f77 is now active with session: 18674125-3bbe-49cd-9b34-e43c80f24062","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.498205888Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
3022025-01-03T08:34:21.405Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.498217968Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228,"session_id":"18674125-3bbe-49cd-9b34-e43c80f24062"}
3032025-01-03T08:34:21.405Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:20.498230258Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228}
3042025-01-03T08:34:21.405ZDisk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
3052025-01-03T08:34:21.405ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
3062025-01-03T08:34:21.405ZFill test
3072025-01-03T08:34:21.441ZRead and Verify all blocks (0..50 range:false)
3082025-01-03T08:34:21.480ZWrote out file "/var/tmp/test_up-build/verify_file"
3092025-01-03T08:34:21.480ZCLIENT: Tests done. All submitted work has been ACK'd
3102025-01-03T08:34:21.483Z----------------------------------------------------------------
3112025-01-03T08:34:21.483Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
3122025-01-03T08:34:21.483ZStates: Active Active Active
3132025-01-03T08:34:21.483ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
3142025-01-03T08:34:21.483Z 1002 Acked Read 50 Done Done Done false
3152025-01-03T08:34:21.483Z STATES DS:0 DS:1 DS:2 TOTAL
3162025-01-03T08:34:21.483Z Sent 0 0 0 0
3172025-01-03T08:34:21.483Z Done 1 1 1 3
3182025-01-03T08:34:21.483Z Skipped 0 0 0 0
3192025-01-03T08:34:21.483Z Error 0 0 0 0
3202025-01-03T08:34:21.483ZLast Flush: 1001 1001 1001
3212025-01-03T08:34:21.483ZDownstairs last five completed: 1001 1000
3222025-01-03T08:34:21.483ZUpstairs last five completed: 1002 1001 1000
3232025-01-03T08:34:21.483ZCLIENT: Up:0 ds:1 act:3
3242025-01-03T08:34:25.482Z----------------------------------------------------------------
3252025-01-03T08:34:25.482Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
3262025-01-03T08:34:25.482ZDownstairs last five completed: 1003 1002 1001 1000
3272025-01-03T08:34:25.482ZUpstairs last five completed: 1003 1002 1001 1000
3282025-01-03T08:34:25.482ZCLIENT: Up:0 ds:0 act:3
3292025-01-03T08:34:25.482ZCLIENT: All crucible jobs finished, exiting program
3302025-01-03T08:34:25.485Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:34:24.579037021Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1228}
3312025-01-03T08:34:25.489ZRepair part 1 passed
3322025-01-03T08:34:25.492Z
3332025-01-03T08:34:25.492ZKill the current downstairs
3342025-01-03T08:34:25.656ZDownstairs 2 stopped
3352025-01-03T08:34:25.656Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
3362025-01-03T08:34:25.659ZNow put back the original so we have a mismatch
3372025-01-03T08:34:25.659Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
3382025-01-03T08:34:25.662ZRestart downstairs with old directory
3392025-01-03T08:34:25.715Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
3402025-01-03T08:34:25.732ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3412025-01-03T08:34:25.734Z 0 000-009 32 32 31 1499 1499 1498 F F F <---
3422025-01-03T08:34:25.734Z 1 010-019 32 32 31 1499 1499 1498 F F F <---
3432025-01-03T08:34:25.734Z 2 020-029 32 32 31 1499 1499 1498 F F F <---
3442025-01-03T08:34:25.734Z 3 030-039 32 32 31 1499 1499 1498 F F F <---
3452025-01-03T08:34:25.734Z 4 040-049 32 32 31 1499 1499 1498 F F F <---
3462025-01-03T08:34:25.734ZMax gen: 32, Max flush: 1499
3472025-01-03T08:34:25.734ZError: Difference in extent metadata found!
3482025-01-03T08:34:25.737Zdump test found error as expected
3492025-01-03T08:34:25.737Z
3502025-01-03T08:34:25.737Z
3512025-01-03T08:34:25.737Z/var/tmp/bins/crutest verify --range -g 33 -q --key 19Vu0KYX6uqE7olrTYPbVXU+Lq93I99hvQcKpwGNpMM= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
3522025-01-03T08:34:25.793Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.890175834Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"task":"crutest"}
3532025-01-03T08:34:25.796Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.890765568Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"task":"crutest"}
3542025-01-03T08:34:25.796Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.892514498Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"task":"crutest"}
3552025-01-03T08:34:25.796Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.892731733Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236}
3562025-01-03T08:34:25.796Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.892771974Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236}
3572025-01-03T08:34:25.796Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.892795365Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236}
3582025-01-03T08:34:25.796Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.892811755Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236}
3592025-01-03T08:34:25.796Z{"msg":"Crucible ab7550d1-74d0-4922-bfe1-67eeba1abbbb has session id: 335b5a83-2669-44f6-a900-206ed92a42f9","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.893285156Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3602025-01-03T08:34:25.796Z{"msg":"Upstairs opts: Upstairs UUID: ab7550d1-74d0-4922-bfe1-67eeba1abbbb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.893342527Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3612025-01-03T08:34:25.797Z{"msg":"Crucible stats registered with UUID: ab7550d1-74d0-4922-bfe1-67eeba1abbbb","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.893369618Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3622025-01-03T08:34:25.831Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.928422526Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"task":"crutest"}
3632025-01-03T08:34:25.831Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.928486528Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236}
3642025-01-03T08:34:25.831Z{"msg":"ab7550d1-74d0-4922-bfe1-67eeba1abbbb active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.928503678Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3652025-01-03T08:34:25.834Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.928529058Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"client":"0","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3662025-01-03T08:34:25.834Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.928633681Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"client":"0","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3672025-01-03T08:34:25.834Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.928696673Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"client":"1","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3682025-01-03T08:34:25.834Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.928732333Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"client":"1","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3692025-01-03T08:34:25.834Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.928774524Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"client":"2","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3702025-01-03T08:34:25.834Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.928794755Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"client":"2","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3712025-01-03T08:34:25.834Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.928821505Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"":"io task","client":"2","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3722025-01-03T08:34:25.834Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.928840216Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"":"io task","client":"0","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3732025-01-03T08:34:25.834Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.928884237Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"":"io task","client":"1","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3742025-01-03T08:34:25.834Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:34:24.928932718Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"":"io task","client":"2","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3752025-01-03T08:34:25.834Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:34:24.928970709Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"":"io task","client":"2","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3762025-01-03T08:34:25.834Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:34:24.92902257Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3772025-01-03T08:34:25.834Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.929144893Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"":"io task","client":"2","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3782025-01-03T08:34:25.834Z{"msg":"ds_connection connected from Ok(127.0.0.1:55386)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.929245015Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"":"io task","client":"1","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3792025-01-03T08:34:25.834Z{"msg":"ds_connection connected from Ok(127.0.0.1:62407)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.929307247Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"":"io task","client":"0","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3802025-01-03T08:34:25.834Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.92989271Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"client":"1","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3812025-01-03T08:34:25.834Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.929954261Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"client":"1","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3822025-01-03T08:34:25.834Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.929996082Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"client":"0","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3832025-01-03T08:34:25.834Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.930011863Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"client":"0","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3842025-01-03T08:34:25.834Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.930567376Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"client":"1","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3852025-01-03T08:34:25.834Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.930603116Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"client":"0","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3862025-01-03T08:34:25.837Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.930976605Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"client":"1","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3872025-01-03T08:34:25.837Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.931020596Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"client":"1","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3882025-01-03T08:34:25.837Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.931038987Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3892025-01-03T08:34:25.837Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.931051977Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"client":"0","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3902025-01-03T08:34:25.838Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.931075637Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"client":"0","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3912025-01-03T08:34:25.838Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:24.931089768Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3922025-01-03T08:34:35.833Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.929616753Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"":"io task","client":"2","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3932025-01-03T08:34:35.833Z{"msg":"ds_connection connected from Ok(127.0.0.1:42537)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.929712726Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"":"io task","client":"2","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3942025-01-03T08:34:35.836Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.930918964Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"client":"2","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3952025-01-03T08:34:35.836Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.930989195Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"client":"2","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3962025-01-03T08:34:35.836Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.931500167Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"client":"2","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3972025-01-03T08:34:35.836Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.931863845Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"client":"2","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3982025-01-03T08:34:35.836Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.931899426Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"client":"2","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
3992025-01-03T08:34:35.836Z{"msg":"[0]R flush_numbers: [1499, 1499, 1499, 1499, 1499]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.931926507Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4002025-01-03T08:34:35.836Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.931955947Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4012025-01-03T08:34:35.836Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.931972148Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4022025-01-03T08:34:35.836Z{"msg":"[1]R flush_numbers: [1499, 1499, 1499, 1499, 1499]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.931994579Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4032025-01-03T08:34:35.836Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932008809Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4042025-01-03T08:34:35.836Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932029949Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4052025-01-03T08:34:35.836Z{"msg":"[2]R flush_numbers: [1498, 1498, 1498, 1498, 1498]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932044129Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4062025-01-03T08:34:35.836Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.93205789Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4072025-01-03T08:34:35.836Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.93207928Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4082025-01-03T08:34:35.836Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932092861Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4092025-01-03T08:34:35.836Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932114501Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4102025-01-03T08:34:35.836Z{"msg":"Next flush: 1500","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932128751Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4112025-01-03T08:34:35.836Z{"msg":"Extent 0 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932150152Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4122025-01-03T08:34:35.836Z{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932165002Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4132025-01-03T08:34:35.836Z{"msg":"extent:0 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932187463Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4142025-01-03T08:34:35.836Z{"msg":"extent:0 flush: 1499 1499 1498 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932203293Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4152025-01-03T08:34:35.837Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932225864Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4162025-01-03T08:34:35.837Z{"msg":"extent:0 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932240644Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4172025-01-03T08:34:35.837Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932263445Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4182025-01-03T08:34:35.837Z{"msg":"find dest for source 0 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932278405Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4192025-01-03T08:34:35.837Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932301175Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4202025-01-03T08:34:35.837Z{"msg":"Extent 1 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932332156Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4212025-01-03T08:34:35.837Z{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932358487Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4222025-01-03T08:34:35.837Z{"msg":"extent:1 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932384808Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4232025-01-03T08:34:35.841Z{"msg":"extent:1 flush: 1499 1499 1498 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932412128Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4242025-01-03T08:34:35.841Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932453389Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4252025-01-03T08:34:35.841Z{"msg":"extent:1 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.93249028Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4262025-01-03T08:34:35.841Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932515871Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4272025-01-03T08:34:35.841Z{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932553841Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4282025-01-03T08:34:35.841Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932593332Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4292025-01-03T08:34:35.841Z{"msg":"Extent 2 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932622773Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4302025-01-03T08:34:35.842Z{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932639003Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4312025-01-03T08:34:35.842Z{"msg":"extent:2 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932660534Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4322025-01-03T08:34:35.842Z{"msg":"extent:2 flush: 1499 1499 1498 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932675704Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4332025-01-03T08:34:35.842Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932706605Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4342025-01-03T08:34:35.842Z{"msg":"extent:2 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932725195Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4352025-01-03T08:34:35.842Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932739916Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4362025-01-03T08:34:35.842Z{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932763006Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4372025-01-03T08:34:35.842Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932779477Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4382025-01-03T08:34:35.842Z{"msg":"Extent 3 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932808227Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4392025-01-03T08:34:35.842Z{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932824907Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4402025-01-03T08:34:35.842Z{"msg":"extent:3 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932852018Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4412025-01-03T08:34:35.842Z{"msg":"extent:3 flush: 1499 1499 1498 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932878839Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4422025-01-03T08:34:35.842Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.9329092Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4432025-01-03T08:34:35.842Z{"msg":"extent:3 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.93295345Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4442025-01-03T08:34:35.842Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.932997411Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4452025-01-03T08:34:35.842Z{"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.933043312Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4462025-01-03T08:34:35.842Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.933086924Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4472025-01-03T08:34:35.842Z{"msg":"Extent 4 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.933107184Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4482025-01-03T08:34:35.842Z{"msg":"First source client ID for extent 4","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.933136835Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4492025-01-03T08:34:35.842Z{"msg":"extent:4 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.933164625Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4502025-01-03T08:34:35.842Z{"msg":"extent:4 flush: 1499 1499 1498 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.933182116Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4512025-01-03T08:34:35.842Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.933212656Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4522025-01-03T08:34:35.842Z{"msg":"extent:4 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.933229417Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4532025-01-03T08:34:35.842Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.933263448Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4542025-01-03T08:34:35.842Z{"msg":"find dest for source 0 for extent at index 4","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.933280418Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4552025-01-03T08:34:35.842Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.933303329Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4562025-01-03T08:34:35.842Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.933319639Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"client":"0","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4572025-01-03T08:34:35.842Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.93334209Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"client":"1","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4582025-01-03T08:34:35.842Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.93335628Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"client":"2","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4592025-01-03T08:34:35.843Z{"msg":"Full repair list: {ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.93338221Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4602025-01-03T08:34:35.843Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 1500, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 1500, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 1500, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 1500, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 1500, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.933461812Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4612025-01-03T08:34:35.843Z{"msg":"starting reconciliation 4bb2b951-1de2-4a8c-a19c-3483f6006458: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.933479193Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4622025-01-03T08:34:35.843Z{"msg":"no Nexus client from DNS, aborting start notification","v":0,"name":"crucible","level":50,"time":"2025-01-03T08:34:34.933503113Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"reconcile":"4bb2b951-1de2-4a8c-a19c-3483f6006458","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4632025-01-03T08:34:35.843Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-03T08:34:34.933532204Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"reconcile":"4bb2b951-1de2-4a8c-a19c-3483f6006458","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4642025-01-03T08:34:35.843Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-03T08:34:34.933560055Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"reconcile":"4bb2b951-1de2-4a8c-a19c-3483f6006458","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4652025-01-03T08:34:35.843Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.933588905Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"client":"2","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4662025-01-03T08:34:35.843Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-03T08:34:34.933620436Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"reconcile":"4bb2b951-1de2-4a8c-a19c-3483f6006458","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4672025-01-03T08:34:35.846Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-03T08:34:34.940366032Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"reconcile":"4bb2b951-1de2-4a8c-a19c-3483f6006458","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4682025-01-03T08:34:35.846Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-03T08:34:34.940761381Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"reconcile":"4bb2b951-1de2-4a8c-a19c-3483f6006458","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4692025-01-03T08:34:35.846Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-03T08:34:34.941032747Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"reconcile":"4bb2b951-1de2-4a8c-a19c-3483f6006458","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4702025-01-03T08:34:35.846Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.941394385Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"client":"2","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4712025-01-03T08:34:35.846Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-03T08:34:34.941449316Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"reconcile":"4bb2b951-1de2-4a8c-a19c-3483f6006458","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4722025-01-03T08:34:35.848Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-03T08:34:34.944393614Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"reconcile":"4bb2b951-1de2-4a8c-a19c-3483f6006458","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4732025-01-03T08:34:35.848Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-03T08:34:34.944771573Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"reconcile":"4bb2b951-1de2-4a8c-a19c-3483f6006458","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4742025-01-03T08:34:35.848Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-03T08:34:34.945011679Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"reconcile":"4bb2b951-1de2-4a8c-a19c-3483f6006458","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4752025-01-03T08:34:35.848Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.945329366Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"client":"2","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4762025-01-03T08:34:35.849Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-03T08:34:34.945363737Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"reconcile":"4bb2b951-1de2-4a8c-a19c-3483f6006458","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4772025-01-03T08:34:35.851Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-03T08:34:34.948187502Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"reconcile":"4bb2b951-1de2-4a8c-a19c-3483f6006458","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4782025-01-03T08:34:35.853Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-03T08:34:34.948673173Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"reconcile":"4bb2b951-1de2-4a8c-a19c-3483f6006458","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4792025-01-03T08:34:35.853Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-03T08:34:34.948898228Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"reconcile":"4bb2b951-1de2-4a8c-a19c-3483f6006458","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4802025-01-03T08:34:35.853Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.949188745Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"client":"2","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4812025-01-03T08:34:35.853Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-03T08:34:34.949229496Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"reconcile":"4bb2b951-1de2-4a8c-a19c-3483f6006458","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4822025-01-03T08:34:35.856Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-03T08:34:34.951960649Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"reconcile":"4bb2b951-1de2-4a8c-a19c-3483f6006458","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4832025-01-03T08:34:35.856Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-03T08:34:34.952372458Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"reconcile":"4bb2b951-1de2-4a8c-a19c-3483f6006458","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4842025-01-03T08:34:35.856Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-03T08:34:34.952601303Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"reconcile":"4bb2b951-1de2-4a8c-a19c-3483f6006458","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4852025-01-03T08:34:35.858Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.95289169Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"client":"2","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4862025-01-03T08:34:35.858Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-03T08:34:34.952945322Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"reconcile":"4bb2b951-1de2-4a8c-a19c-3483f6006458","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4872025-01-03T08:34:35.860Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-03T08:34:34.955804077Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"reconcile":"4bb2b951-1de2-4a8c-a19c-3483f6006458","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4882025-01-03T08:34:35.860Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.956201017Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4892025-01-03T08:34:35.860Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.956239007Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"client":"0","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4902025-01-03T08:34:35.860Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.956255758Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"client":"1","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4912025-01-03T08:34:35.860Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.956278348Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"client":"2","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4922025-01-03T08:34:35.861Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.956299459Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4932025-01-03T08:34:35.861Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.956312279Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4942025-01-03T08:34:35.861Z{"msg":"ab7550d1-74d0-4922-bfe1-67eeba1abbbb is now active with session: 335b5a83-2669-44f6-a900-206ed92a42f9","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.956326449Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4952025-01-03T08:34:35.861Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:34.95634749Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236}
4962025-01-03T08:34:35.861Z{"msg":"no Nexus client from DNS, aborting finish notification","v":0,"name":"crucible","level":50,"time":"2025-01-03T08:34:34.9563604Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236,"reconcile":"4bb2b951-1de2-4a8c-a19c-3483f6006458","":"downstairs","":"downstairs","session_id":"335b5a83-2669-44f6-a900-206ed92a42f9"}
4972025-01-03T08:34:35.861ZWait for a query_work_queue command to finish before sending IO
4982025-01-03T08:34:35.861ZDisk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
4992025-01-03T08:34:35.861ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
5002025-01-03T08:34:35.861ZRead and Verify all blocks (0..50 range:true)
5012025-01-03T08:34:35.900ZWrote out file "/var/tmp/test_up-build/verify_file"
5022025-01-03T08:34:35.900ZVerify test completed
5032025-01-03T08:34:35.903ZWrote out file "/var/tmp/test_up-build/verify_file"
5042025-01-03T08:34:35.903ZCLIENT: Tests done. All submitted work has been ACK'd
5052025-01-03T08:34:35.903Z----------------------------------------------------------------
5062025-01-03T08:34:35.903Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
5072025-01-03T08:34:35.903ZStates: Active Active Active
5082025-01-03T08:34:35.903ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
5092025-01-03T08:34:35.903Z 1000 Acked Read 50 Done Done Done false
5102025-01-03T08:34:35.903Z STATES DS:0 DS:1 DS:2 TOTAL
5112025-01-03T08:34:35.903Z Sent 0 0 0 0
5122025-01-03T08:34:35.903Z Done 1 1 1 3
5132025-01-03T08:34:35.903Z Skipped 0 0 0 0
5142025-01-03T08:34:35.903Z Error 0 0 0 0
5152025-01-03T08:34:35.903ZLast Flush: 0 0 0
5162025-01-03T08:34:35.903ZDownstairs last five completed:
5172025-01-03T08:34:35.903ZUpstairs last five completed: 1000
5182025-01-03T08:34:35.903ZCLIENT: Up:0 ds:1 act:3
5192025-01-03T08:34:39.901Z----------------------------------------------------------------
5202025-01-03T08:34:39.901Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
5212025-01-03T08:34:39.901ZDownstairs last five completed: 1001 1000
5222025-01-03T08:34:39.901ZUpstairs last five completed: 1001 1000
5232025-01-03T08:34:39.901ZCLIENT: Up:0 ds:0 act:3
5242025-01-03T08:34:39.901ZCLIENT: All crucible jobs finished, exiting program
5252025-01-03T08:34:39.904Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:34:38.998375609Z","hostname":"w-01JGNMH822F3VW2FFS29KX8E6C","pid":1236}
5262025-01-03T08:34:39.908ZRepair part 2 passed
5272025-01-03T08:34:39.911Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
5282025-01-03T08:34:39.925ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
5292025-01-03T08:34:39.925Z 0 000-009 32 32 32 1499 1499 1499 F F F
5302025-01-03T08:34:39.927Z 1 010-019 32 32 32 1499 1499 1499 F F F
5312025-01-03T08:34:39.927Z 2 020-029 32 32 32 1499 1499 1499 F F F
5322025-01-03T08:34:39.927Z 3 030-039 32 32 32 1499 1499 1499 F F F
5332025-01-03T08:34:39.927Z 4 040-049 32 32 32 1499 1499 1499 F F F
5342025-01-03T08:34:39.927ZMax gen: 32, Max flush: 1499
5352025-01-03T08:34:39.930Zdump test passed
5362025-01-03T08:34:39.930Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
5372025-01-03T08:34:39.947Z Extent 1
5382025-01-03T08:34:39.947ZGEN 32 32 32
5392025-01-03T08:34:39.947ZFLUSH_ID 1499 1499 1499
5402025-01-03T08:34:39.947ZDIRTY
5412025-01-03T08:34:39.947Z
5422025-01-03T08:34:39.947ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
5432025-01-03T08:34:39.950Z010 A A A A A A
5442025-01-03T08:34:39.950Z011 A A A A A A
5452025-01-03T08:34:39.952Z012 A A A A A A
5462025-01-03T08:34:39.952Z013 A A A A A A
5472025-01-03T08:34:39.954Z014 A A A A A A
5482025-01-03T08:34:39.957Z015 A A A A A A
5492025-01-03T08:34:39.957Z016 A A A A A A
5502025-01-03T08:34:39.959Z017 A A A A A A
5512025-01-03T08:34:39.959Z018 A A A A A A
5522025-01-03T08:34:39.962Z019 A A A A A A
5532025-01-03T08:34:39.964Zdump extent test passed
5542025-01-03T08:34:39.966Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
5552025-01-03T08:34:39.980ZExtent 2 Block in extent 0 Actual block 20
5562025-01-03T08:34:39.980Z
5572025-01-03T08:34:39.982Z DATA SHA256 VER
5582025-01-03T08:34:39.982Z------ ---------------------------------------------------------------- ---
5592025-01-03T08:34:39.982Z 0 2d0c8ad27d734d3797b6a6edd9ab3ac7a1d1b2cfe342aa3d513b79876eb39c1b A
5602025-01-03T08:34:39.982Z 1 2d0c8ad27d734d3797b6a6edd9ab3ac7a1d1b2cfe342aa3d513b79876eb39c1b A
5612025-01-03T08:34:39.985Z 2 2d0c8ad27d734d3797b6a6edd9ab3ac7a1d1b2cfe342aa3d513b79876eb39c1b A
5622025-01-03T08:34:39.985Z
5632025-01-03T08:34:39.985ZNONCES 0 1 2 DIFF
5642025-01-03T08:34:39.985Z------ ------------------------ ------------------------ ------------------------ -----
5652025-01-03T08:34:39.985Z 0 89810a8c115f4cf7909880af 89810a8c115f4cf7909880af 89810a8c115f4cf7909880af
5662025-01-03T08:34:39.985Z
5672025-01-03T08:34:39.985Z TAGS 0 1 2 DIFF
5682025-01-03T08:34:39.985Z------ -------------------------------- -------------------------------- -------------------------------- -----
5692025-01-03T08:34:39.985Z 0 459983959b96eac02a40286ec15ca23c 459983959b96eac02a40286ec15ca23c 459983959b96eac02a40286ec15ca23c
5702025-01-03T08:34:39.985Z
5712025-01-03T08:34:39.985ZHASHES 0 1 2 DIFF
5722025-01-03T08:34:39.985Z------ ---------------- ---------------- ---------------- -----
5732025-01-03T08:34:39.985Z
5742025-01-03T08:34:39.987Zdump block test passed
5752025-01-03T08:34:39.987ZInitial upstairs tests have completed, stopping all downstairs
5762025-01-03T08:34:45.051ZCreating 4 larger downstairs regions
5772025-01-03T08:34:45.169ZStarting 4 downstairs
5782025-01-03T08:34:50.176Zdsc restarted at PID: 1249
5792025-01-03T08:34:50.402ZNow do the replace-reconcile test
5802025-01-03T08:34:50.457ZUsing 8840 for the replacement port
5812025-01-03T08:37:41.628ZCompleted test: replace-reconcile
5822025-01-03T08:37:41.630ZNow do the replace-before-active test
5832025-01-03T08:40:48.164ZCompleted test: replace-before-active
5842025-01-03T08:40:48.164ZAll tests have completed, stopping all downstairs
5852025-01-03T08:40:48.223Z
5862025-01-03T08:40:48.223ZAll Tests have passed
5872025-01-03T08:40:48.226Z9:52 Test duration
5882025-01-03T08:40:48.226Z
5892025-01-03T08:40:48.226Zreal 9:51.374154877
5902025-01-03T08:40:48.226Zuser 6:24.779584449
5912025-01-03T08:40:48.226Zsys 12.335595369
5922025-01-03T08:40:48.226Ztrap 0.206534373
5932025-01-03T08:40:48.226Ztflt 0.014318408
5942025-01-03T08:40:48.226Zdflt 0.080853723
5952025-01-03T08:40:48.226Zkflt 0.001234649
5962025-01-03T08:40:48.226Zlock 8:29:06.557488140
5972025-01-03T08:40:48.226Zslp 33:53.276704765
5982025-01-03T08:40:48.226Zlat 24.636568346
5992025-01-03T08:40:48.226Zstop 0.134318601
6002025-01-03T08:40:48.226Z+ echo 'test-up-encrypted ends'
6012025-01-03T08:40:48.226Ztest-up-encrypted ends
6022025-01-03T08:40:53.229Zprocess exited: duration 599567 ms, exit code 0
6032025-01-03T08:40:53.230Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
6042025-01-03T08:41:53.273Zexec warning: : stdout descriptor may be held open by a background process; giving up!
6052025-01-03T08:41:53.273Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
6062025-01-03T08:41:53.284Zfound 14 output files
6072025-01-03T08:41:53.284Zuploading: /tmp/test_up-build/dsc-out.txt (39489 bytes)
6082025-01-03T08:41:54.299Zuploaded: /tmp/test_up-build/dsc-out.txt
6092025-01-03T08:41:54.307Zuploading: /tmp/test_up-build/test_up_out.txt (10395724 bytes)
6102025-01-03T08:41:55.447Zuploaded: /tmp/test_up-build/test_up_out.txt
6112025-01-03T08:41:55.450Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (2507 bytes)
6122025-01-03T08:41:56.455Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
6132025-01-03T08:41:56.457Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (144663 bytes)
6142025-01-03T08:41:57.470Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
6152025-01-03T08:41:57.473Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (251112 bytes)
6162025-01-03T08:41:58.489Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
6172025-01-03T08:41:58.492Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (139439 bytes)
6182025-01-03T08:41:59.504Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
6192025-01-03T08:41:59.507Zuploading: /tmp/debug/df.txt (1270 bytes)
6202025-01-03T08:42:00.512Zuploaded: /tmp/debug/df.txt
6212025-01-03T08:42:00.514Zuploading: /tmp/debug/dtrace.txt (471587 bytes)
6222025-01-03T08:42:00.537Zupload warning: file "/tmp/debug/dtrace.txt" changed size mid upload: 471587 -> 558026
6232025-01-03T08:42:01.542Zuploaded: /tmp/debug/dtrace.txt
6242025-01-03T08:42:01.545Zuploading: /tmp/debug/iostat.txt (197131 bytes)
6252025-01-03T08:42:01.556Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 197131 -> 199531
6262025-01-03T08:42:02.562Zuploaded: /tmp/debug/iostat.txt
6272025-01-03T08:42:02.564Zuploading: /tmp/debug/mpstat.txt (478960 bytes)
6282025-01-03T08:42:02.585Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 478960 -> 485521
6292025-01-03T08:42:03.590Zuploaded: /tmp/debug/mpstat.txt
6302025-01-03T08:42:03.593Zuploading: /tmp/debug/paging.txt (85983 bytes)
6312025-01-03T08:42:03.599Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 85983 -> 87291
6322025-01-03T08:42:04.604Zuploaded: /tmp/debug/paging.txt
6332025-01-03T08:42:04.604Zuploading: /tmp/debug/prstat.txt (918920 bytes)
6342025-01-03T08:42:04.637Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 918920 -> 933129
6352025-01-03T08:42:05.642Zuploaded: /tmp/debug/prstat.txt
6362025-01-03T08:42:05.644Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
6372025-01-03T08:42:06.649Zuploaded: /tmp/debug/psrinfo.txt
6382025-01-03T08:42:06.652Zuploading: /tmp/debug/upstairs-info.txt (94240 bytes)
6392025-01-03T08:42:07.663Zuploaded: /tmp/debug/upstairs-info.txt