01JH8G7RR403PK7G42DD105PPT: test-up-unencrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01JH8G81H3K5MHTB71E74SPJ3F

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12025-01-10T16:33:46.932Zjob dependencies complete; ready to run (waiting for 13 m 41 s)
22025-01-10T16:34:50.115Zjob assigned to worker 01JH8H1HV4PDNPGET5JKDVHX88 [factory aws, i-0594c7475fa69ca4f] (queued for 1 m 3 s)
32025-01-10T16:34:57.483Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42025-01-10T16:34:58.592Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52025-01-10T16:34:58.592Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62025-01-10T16:34:59.654Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72025-01-10T16:34:59.654Zdownloading input: /input/build/work/bins/crutest.gz
82025-01-10T16:35:01.064Zdownloaded input: /input/build/work/bins/crutest.gz
92025-01-10T16:35:01.064Zdownloading input: /input/build/work/bins/dsc.gz
102025-01-10T16:35:01.802Zdownloaded input: /input/build/work/bins/dsc.gz
112025-01-10T16:35:01.805Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122025-01-10T16:35:01.822Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132025-01-10T16:35:01.822Zdownloading input: /input/build/work/scripts/test_ds.sh
142025-01-10T16:35:01.838Zdownloaded input: /input/build/work/scripts/test_ds.sh
152025-01-10T16:35:01.838Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162025-01-10T16:35:01.858Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172025-01-10T16:35:01.858Zdownloading input: /input/build/work/scripts/test_repair.sh
182025-01-10T16:35:01.884Zdownloaded input: /input/build/work/scripts/test_repair.sh
192025-01-10T16:35:01.884Zdownloading input: /input/build/work/scripts/test_replay.sh
202025-01-10T16:35:01.908Zdownloaded input: /input/build/work/scripts/test_replay.sh
212025-01-10T16:35:01.908Zdownloading input: /input/build/work/scripts/test_up.sh
222025-01-10T16:35:01.935Zdownloaded input: /input/build/work/scripts/test_up.sh
232025-01-10T16:35:01.935Zdownloading input: /input/build/work/scripts/upstairs_info.d
242025-01-10T16:35:01.958Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252025-01-10T16:35:01.962Zdownloading input: /input/build/tmp/cargo-test-out.log
262025-01-10T16:35:02.679Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272025-01-10T16:35:02.679Zstarting task 0: "setup"
282025-01-10T16:35:02.692Z++ uname -s
292025-01-10T16:35:02.694Z+ kern=SunOS
302025-01-10T16:35:02.694Z+ case "$kern" in
312025-01-10T16:35:02.694Z+ groupadd -g 12345 build
322025-01-10T16:35:02.697Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332025-01-10T16:35:04.705Z+ zfs create -o mountpoint=/work rpool/work
342025-01-10T16:35:04.910Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352025-01-10T16:35:04.912Z+ home_fs=zfs
362025-01-10T16:35:04.912Z+ [[ zfs == autofs ]]
372025-01-10T16:35:04.913Z+ mkdir -p /home/build
382025-01-10T16:35:04.915Z+ chown build:build /home/build /work
392025-01-10T16:35:06.917Z+ chmod 0700 /home/build /work
402025-01-10T16:35:06.919Zprocess exited: duration 4240 ms, exit code 0
 
412025-01-10T16:35:06.925Zstarting task 1: "authentication"
422025-01-10T16:35:06.943Zprocess exited: duration 17 ms, exit code 0
 
432025-01-10T16:35:06.949Zstarting task 2: "build"
442025-01-10T16:35:06.955Z+ banner cores
452025-01-10T16:35:06.958Z
462025-01-10T16:35:06.958Z #### #### ##### ###### ####
472025-01-10T16:35:06.958Z # # # # # # # #
482025-01-10T16:35:06.958Z # # # # # ##### ####
492025-01-10T16:35:06.958Z # # # ##### # #
502025-01-10T16:35:06.958Z # # # # # # # # #
512025-01-10T16:35:06.958Z #### #### # # ###### ####
522025-01-10T16:35:06.958Z
532025-01-10T16:35:06.960Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542025-01-10T16:35:06.966Z+ echo 'input bins dir contains:'
552025-01-10T16:35:06.966Zinput bins dir contains:
562025-01-10T16:35:06.969Z+ ls -ltr /input/build/work/bins
572025-01-10T16:35:06.969Ztotal 762721
582025-01-10T16:35:06.969Z-rw-r--r-- 1 root root 109568203 Jan 10 16:34 crucible-downstairs.gz
592025-01-10T16:35:06.969Z-rw-r--r-- 1 root root 105130075 Jan 10 16:34 crucible-hammer.gz
602025-01-10T16:35:06.969Z-rw-r--r-- 1 root root 109591730 Jan 10 16:35 crutest.gz
612025-01-10T16:35:06.969Z-rw-r--r-- 1 root root 65965887 Jan 10 16:35 dsc.gz
622025-01-10T16:35:06.969Z+ banner unpack
632025-01-10T16:35:06.972Z
642025-01-10T16:35:06.972Z # # # # ##### ## #### # #
652025-01-10T16:35:06.972Z # # ## # # # # # # # # #
662025-01-10T16:35:06.972Z # # # # # # # # # # ####
672025-01-10T16:35:06.972Z # # # # # ##### ###### # # #
682025-01-10T16:35:06.972Z # # # ## # # # # # # #
692025-01-10T16:35:06.972Z #### # # # # # #### # #
702025-01-10T16:35:06.972Z
712025-01-10T16:35:06.972Z+ mkdir -p /var/tmp/bins
722025-01-10T16:35:06.975Z+ for t in "$input/bins/"*.gz
732025-01-10T16:35:06.975Z++ basename /input/build/work/bins/crucible-downstairs.gz
742025-01-10T16:35:06.975Z+ b=crucible-downstairs.gz
752025-01-10T16:35:06.975Z+ b=crucible-downstairs
762025-01-10T16:35:06.975Z+ gunzip
772025-01-10T16:35:09.227Z+ chmod +x /var/tmp/bins/crucible-downstairs
782025-01-10T16:35:09.229Z+ for t in "$input/bins/"*.gz
792025-01-10T16:35:09.232Z++ basename /input/build/work/bins/crucible-hammer.gz
802025-01-10T16:35:09.232Z+ b=crucible-hammer.gz
812025-01-10T16:35:09.232Z+ b=crucible-hammer
822025-01-10T16:35:09.232Z+ gunzip
832025-01-10T16:35:11.380Z+ chmod +x /var/tmp/bins/crucible-hammer
842025-01-10T16:35:11.383Z+ for t in "$input/bins/"*.gz
852025-01-10T16:35:11.385Z++ basename /input/build/work/bins/crutest.gz
862025-01-10T16:35:11.385Z+ b=crutest.gz
872025-01-10T16:35:11.385Z+ b=crutest
882025-01-10T16:35:11.385Z+ gunzip
892025-01-10T16:35:13.702Z+ chmod +x /var/tmp/bins/crutest
902025-01-10T16:35:13.705Z+ for t in "$input/bins/"*.gz
912025-01-10T16:35:13.707Z++ basename /input/build/work/bins/dsc.gz
922025-01-10T16:35:13.709Z+ b=dsc.gz
932025-01-10T16:35:13.709Z+ b=dsc
942025-01-10T16:35:13.709Z+ gunzip
952025-01-10T16:35:15.085Z+ chmod +x /var/tmp/bins/dsc
962025-01-10T16:35:15.088Z+ export BINDIR=/var/tmp/bins
972025-01-10T16:35:15.088Z+ BINDIR=/var/tmp/bins
982025-01-10T16:35:15.088Z+ jobpid=1139
992025-01-10T16:35:15.090Z+ echo 'Setup debug logging'
1002025-01-10T16:35:15.091ZSetup debug logging
1012025-01-10T16:35:15.091Z+ mkdir /tmp/debug
1022025-01-10T16:35:15.091Z+ sleep 7200
1032025-01-10T16:35:15.091Z+ psrinfo -v
1042025-01-10T16:35:15.094Z+ df -h
1052025-01-10T16:35:15.102Z+ prstat -d d -mLc 1
1062025-01-10T16:35:15.105Z+ iostat -T d -xn 1
1072025-01-10T16:35:15.105Z+ mpstat -T d 1
1082025-01-10T16:35:15.105Z+ vmstat -T d -p 1
1092025-01-10T16:35:15.105Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1102025-01-10T16:35:15.107Z+ banner test_up_unencrypted
1112025-01-10T16:35:15.107Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1122025-01-10T16:35:15.108Z
1132025-01-10T16:35:15.108Z ##### ###### #### ##### # # ##### # # # #
1142025-01-10T16:35:15.108Z # # # # # # # # # # ## #
1152025-01-10T16:35:15.108Z # ##### #### # # # # # # # # # #
1162025-01-10T16:35:15.108Z # # # # # # ##### # # # # #
1172025-01-10T16:35:15.108Z # # # # # # # # # # # ##
1182025-01-10T16:35:15.108Z # ###### #### # ####### #### # ####### #### # #
1192025-01-10T16:35:15.108Z
1202025-01-10T16:35:15.111Z+ ptime -m bash /input/build/work/scripts/test_up.sh -N unencrypted
1212025-01-10T16:35:15.120Z/input/build/work
1222025-01-10T16:35:15.133ZTurn off color for downstairs dump
1232025-01-10T16:35:15.136Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1242025-01-10T16:35:15.136ZCreating 3 downstairs regions
1252025-01-10T16:35:15.464ZStarting 3 downstairs
1262025-01-10T16:35:15.467Zdsc started at PID: 1184
1272025-01-10T16:35:20.651ZDisable automatic restart on all downstairs
1282025-01-10T16:35:20.706Z
1292025-01-10T16:35:20.706ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1302025-01-10T16:35:20.710ZRunning test: span
1312025-01-10T16:35:24.887ZCompleted test: span
1322025-01-10T16:35:29.894ZRunning test: big
1332025-01-10T16:35:34.074ZCompleted test: big
1342025-01-10T16:35:39.081ZRunning test: dep
1352025-01-10T16:35:51.840ZCompleted test: dep
1362025-01-10T16:35:56.848ZRunning test: balloon
1372025-01-10T16:36:06.545ZCompleted test: balloon
1382025-01-10T16:36:11.552ZRunning test: deactivate
1392025-01-10T16:37:05.803ZCompleted test: deactivate
1402025-01-10T16:37:10.811ZRunning hammer
1412025-01-10T16:37:18.914ZRun repair tests
1422025-01-10T16:37:18.918Z/var/tmp/bins/crutest fill -g 31 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1432025-01-10T16:37:18.987Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.032382525Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"task":"crutest"}
1442025-01-10T16:37:18.990Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.032954343Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"task":"crutest"}
1452025-01-10T16:37:18.990Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.034598187Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"task":"crutest"}
1462025-01-10T16:37:18.990Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.034809909Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221}
1472025-01-10T16:37:18.990Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56068760771d58f96ac946e6b2ae83073c18c1b4\",\n git_commit_timestamp: \"2025-01-10T16:19:36.000000000Z\",\n git_branch: \"alan/rust-1-84-please\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.034855107Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221}
1482025-01-10T16:37:18.990Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.034882056Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221}
1492025-01-10T16:37:18.990Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.034900685Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221}
1502025-01-10T16:37:18.990Z{"msg":"Crucible 7f754e06-a444-4220-ab9a-adc4e0de62d9 has session id: d6b8b368-78be-4041-8e7e-87122a9e4777","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.035512531Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1512025-01-10T16:37:18.990Z{"msg":"Upstairs opts: Upstairs UUID: 7f754e06-a444-4220-ab9a-adc4e0de62d9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.035722102Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1522025-01-10T16:37:18.990Z{"msg":"Crucible stats registered with UUID: 7f754e06-a444-4220-ab9a-adc4e0de62d9","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.035757901Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1532025-01-10T16:37:19.017Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.070756698Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"task":"crutest"}
1542025-01-10T16:37:19.017Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.070848164Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221}
1552025-01-10T16:37:19.017Z{"msg":"7f754e06-a444-4220-ab9a-adc4e0de62d9 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.070868293Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1562025-01-10T16:37:19.017Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.070893342Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"client":"0","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1572025-01-10T16:37:19.021Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.070962909Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"client":"0","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1582025-01-10T16:37:19.021Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.071045846Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"client":"1","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1592025-01-10T16:37:19.021Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.071116623Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"client":"1","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1602025-01-10T16:37:19.021Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.071147142Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"client":"2","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1612025-01-10T16:37:19.021Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.071175141Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"client":"2","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1622025-01-10T16:37:19.021Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.071217389Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"":"io task","client":"2","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1632025-01-10T16:37:19.021Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.071260378Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"":"io task","client":"1","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1642025-01-10T16:37:19.021Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.071297306Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"":"io task","client":"0","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1652025-01-10T16:37:19.021Z{"msg":"ds_connection connected from Ok(127.0.0.1:50820)","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.071330285Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"":"io task","client":"1","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1662025-01-10T16:37:19.021Z{"msg":"ds_connection connected from Ok(127.0.0.1:56096)","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.071641922Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"":"io task","client":"2","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1672025-01-10T16:37:19.021Z{"msg":"ds_connection connected from Ok(127.0.0.1:39748)","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.071673011Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"":"io task","client":"0","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1682025-01-10T16:37:19.021Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.072174201Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"client":"2","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1692025-01-10T16:37:19.022Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.072226159Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"client":"2","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1702025-01-10T16:37:19.022Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.072258888Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"client":"1","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1712025-01-10T16:37:19.022Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.072316076Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"client":"1","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1722025-01-10T16:37:19.022Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.072358294Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"client":"0","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1732025-01-10T16:37:19.022Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.072408382Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"client":"0","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1742025-01-10T16:37:19.022Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.073084005Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"client":"0","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1752025-01-10T16:37:19.022Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.073123133Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"client":"2","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1762025-01-10T16:37:19.022Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.073139743Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"client":"1","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1772025-01-10T16:37:19.022Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.073559646Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"client":"2","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1782025-01-10T16:37:19.022Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.073622404Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"client":"2","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1792025-01-10T16:37:19.022Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.073656682Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1802025-01-10T16:37:19.022Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.073679181Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"client":"1","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1812025-01-10T16:37:19.022Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.07370877Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"client":"1","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1822025-01-10T16:37:19.022Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.073732969Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1832025-01-10T16:37:19.022Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.073761208Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"client":"0","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1842025-01-10T16:37:19.022Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.073784887Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"client":"0","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1852025-01-10T16:37:19.022Z{"msg":"[0]R flush_numbers: [1496, 1496, 1496, 1496, 1496]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.073808296Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"":"downstairs","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1862025-01-10T16:37:19.022Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.073832855Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"":"Wait for a query_work_queue command to finish before sending IO
1872025-01-10T16:37:19.022Zdownstairs","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1882025-01-10T16:37:19.022Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.073874043Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"":"downstairs","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1892025-01-10T16:37:19.022Z{"msg":"[1]R flush_numbers: [1496, 1496, 1496, 1496, 1496]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.073898482Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"":"downstairs","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1902025-01-10T16:37:19.022Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.073938461Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"":"downstairs","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1912025-01-10T16:37:19.022Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.07396057Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"":"downstairs","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1922025-01-10T16:37:19.022Z{"msg":"[2]R flush_numbers: [1496, 1496, 1496, 1496, 1496]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.073974499Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"":"downstairs","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1932025-01-10T16:37:19.022Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.073999938Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"":"downstairs","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1942025-01-10T16:37:19.022Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.074024628Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"":"downstairs","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1952025-01-10T16:37:19.022Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.074053446Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"":"downstairs","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1962025-01-10T16:37:19.022Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.074067156Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"":"downstairs","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1972025-01-10T16:37:19.022Z{"msg":"Next flush: 1497","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.074096935Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"":"downstairs","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
1982025-01-10T16:37:19.022Z{"msg":"All extents match","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
1992025-01-10T16:37:19.022Z,"time":"2025-01-10T16:37:18.074115714Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"":"downstairs","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
2002025-01-10T16:37:19.022Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.074131333Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
2012025-01-10T16:37:19.022Z{"msg":"Transition from WaitQuorum to Active","vFill test
2022025-01-10T16:37:19.022Z":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.074155852Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"client":"0","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
2032025-01-10T16:37:19.022Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.074171142Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"client":"1","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
2042025-01-10T16:37:19.023Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.074191121Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"client":"2","":"downstairs","session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
2052025-01-10T16:37:19.023Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.07420403Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
2062025-01-10T16:37:19.023Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.07421607Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
2072025-01-10T16:37:19.023Z{"msg":"7f754e06-a444-4220-ab9a-adc4e0de62d9 is now active with session: d6b8b368-78be-4041-8e7e-87122a9e4777","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.074236879Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
2082025-01-10T16:37:19.023Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.074249099Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221,"session_id":"d6b8b368-78be-4041-8e7e-87122a9e4777"}
2092025-01-10T16:37:19.023Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:18.074267838Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221}
2102025-01-10T16:37:19.026ZRead and Verify all blocks (0..50 range:false)
2112025-01-10T16:37:19.030ZWrote out file "/var/tmp/test_up-build/verify_file"
2122025-01-10T16:37:19.030ZCLIENT: Tests done. All submitted work has been ACK'd
2132025-01-10T16:37:19.034Z----------------------------------------------------------------
2142025-01-10T16:37:19.034Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2152025-01-10T16:37:19.034ZStates: Active Active Active
2162025-01-10T16:37:19.034ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2172025-01-10T16:37:19.034Z 1002 Acked Read 50 Done Done Done false
2182025-01-10T16:37:19.034Z STATES DS:0 DS:1 DS:2 TOTAL
2192025-01-10T16:37:19.034Z Sent 0 0 0 0
2202025-01-10T16:37:19.034Z Done 1 1 1 3
2212025-01-10T16:37:19.034Z Skipped 0 0 0 0
2222025-01-10T16:37:19.034Z Error 0 0 0 0
2232025-01-10T16:37:19.034ZLast Flush: 1001 1001 1001
2242025-01-10T16:37:19.034ZDownstairs last five completed: 1001 1000
2252025-01-10T16:37:19.034ZUpstairs last five completed: 1002 1001 1000
2262025-01-10T16:37:19.034ZCLIENT: Up:0 ds:1 act:3
2272025-01-10T16:37:23.032Z----------------------------------------------------------------
2282025-01-10T16:37:23.033Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2292025-01-10T16:37:23.033ZDownstairs last five completed: 1003 1002 1001 1000
2302025-01-10T16:37:23.033ZUpstairs last five completed: 1003 1002 1001 1000
2312025-01-10T16:37:23.033ZCLIENT: Up:0 ds:0 act:3
2322025-01-10T16:37:23.033ZCLIENT: All crucible jobs finished, exiting program
2332025-01-10T16:37:23.035Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-10T16:37:22.087330104Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1221}
2342025-01-10T16:37:23.044ZRepair setup passed
2352025-01-10T16:37:23.047ZCopy the region for /var/tmp/test_up-build/8830
2362025-01-10T16:37:23.050Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2372025-01-10T16:37:23.052Z/var/tmp/bins/crutest fill -g 32 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2382025-01-10T16:37:23.111Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.165231992Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"task":"crutest"}
2392025-01-10T16:37:23.113Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.165960213Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"task":"crutest"}
2402025-01-10T16:37:23.113Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.167668835Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"task":"crutest"}
2412025-01-10T16:37:23.113Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.167872417Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227}
2422025-01-10T16:37:23.113Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56068760771d58f96ac946e6b2ae83073c18c1b4\",\n git_commit_timestamp: \"2025-01-10T16:19:36.000000000Z\",\n git_branch: \"alan/rust-1-84-please\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.167932395Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227}
2432025-01-10T16:37:23.113Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.167957644Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227}
2442025-01-10T16:37:23.113Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.167976463Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227}
2452025-01-10T16:37:23.116Z{"msg":"Crucible 49b48452-2fc8-405e-8753-275c5f5f6a7e has session id: c2af369d-f397-4ef7-9bd4-3b9e524320fb","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.168540681Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2462025-01-10T16:37:23.116Z{"msg":"Upstairs opts: Upstairs UUID: 49b48452-2fc8-405e-8753-275c5f5f6a7e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.168586769Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2472025-01-10T16:37:23.116Z{"msg":"Crucible stats registered with UUID: 49b48452-2fc8-405e-8753-275c5f5f6a7e","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.168602948Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2482025-01-10T16:37:23.149Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.203906043Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"task":"crutest"}
2492025-01-10T16:37:23.149Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.20398156Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227}
2502025-01-10T16:37:23.149Z{"msg":"49b48452-2fc8-405e-8753-275c5f5f6a7e active request set","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.204011649Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2512025-01-10T16:37:23.154Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.204040827Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"client":"0","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2522025-01-10T16:37:23.154Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.204097945Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"client":"0","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2532025-01-10T16:37:23.154Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.204144653Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"client":"1","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2542025-01-10T16:37:23.154Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.204180952Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"client":"1","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2552025-01-10T16:37:23.154Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.20421417Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"client":"2","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2562025-01-10T16:37:23.154Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.204255649Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"client":"2","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2572025-01-10T16:37:23.154Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.204276168Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"":"io task","client":"0","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2582025-01-10T16:37:23.154Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.204304417Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"":"io task","client":"2","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2592025-01-10T16:37:23.154Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.204332826Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"":"io task","client":"1","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2602025-01-10T16:37:23.154Z{"msg":"ds_connection connected from Ok(127.0.0.1:43430)","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.204359625Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"":"io task","client":"0","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2612025-01-10T16:37:23.154Z{"msg":"ds_connection connected from Ok(127.0.0.1:38212)","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.204619404Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"":"io task","client":"1","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2622025-01-10T16:37:23.154Z{"msg":"ds_connection connected from Ok(127.0.0.1:45605)","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.204667603Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"":"io task","client":"2","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2632025-01-10T16:37:23.154Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.205126344Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"client":"2","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2642025-01-10T16:37:23.154Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.205161513Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"client":"2","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2652025-01-10T16:37:23.154Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.205176612Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"client":"0","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2662025-01-10T16:37:23.154Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.205200371Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"client":"0","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2672025-01-10T16:37:23.155Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.205219581Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"client":"1","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2682025-01-10T16:37:23.155Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.205252599Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"client":"1","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2692025-01-10T16:37:23.155Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.205821006Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"client":"1","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2702025-01-10T16:37:23.155Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.205856845Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"client":"2","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2712025-01-10T16:37:23.155Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.205929932Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"client":"0","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2722025-01-10T16:37:23.155Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.206311867Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"client":"1","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2732025-01-10T16:37:23.155Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.206351815Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"client":"1","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2742025-01-10T16:37:23.155Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.206366605Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2752025-01-10T16:37:23.155Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.206386574Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"client":"2","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2762025-01-10T16:37:23.155Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.206400763Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"client":"2","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2772025-01-10T16:37:23.155Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.206415313Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2782025-01-10T16:37:23.155Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.206434292Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"client":"0","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2792025-01-10T16:37:23.155Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.206447241Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"client":"0","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2802025-01-10T16:37:23.155Z{"msg":"[0]R flush_numbers: [1497, 1497, 1497, 1497, 1497]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.206460081Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"":"downstairs","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2812025-01-10T16:37:23.155Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.206473531Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"":"downstairs","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2822025-01-10T16:37:23.155Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.20649341Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"":"downstairs","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2832025-01-10T16:37:23.155Z{"msg":"[1]R flush_numbers: [1497, 1497, 1497, 1497, 1497]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.206507189Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"":"downstairs","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2842025-01-10T16:37:23.155Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.206526678Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"":"downstairs","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2852025-01-10T16:37:23.155Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.206539678Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"":"downstairs","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2862025-01-10T16:37:23.155ZWait for a query_work_queue command to finish before sending IO
2872025-01-10T16:37:23.155Z{"msg":"[2]R flush_numbers: [1497, 1497, 1497, 1497, 1497]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.206571317Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"":"downstairs","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2882025-01-10T16:37:23.155Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.206584646Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"":"downstairs","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2892025-01-10T16:37:23.155Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.206597295Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"":"downstairs","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2902025-01-10T16:37:23.155Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.206616615Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"":"downstairs","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2912025-01-10T16:37:23.155Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.206629494Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"":"downstairs","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2922025-01-10T16:37:23.155Z{"msg":"Next flush: 1498","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.206642434Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"":"downstairs","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2932025-01-10T16:37:23.155Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.206662573Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"":"downstairs","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2942025-01-10T16:37:23.155Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.206675452Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2952025-01-10T16:37:23.155Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.206687142Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"client":"0","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2962025-01-10T16:37:23.155Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.206699951Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"client":"1","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2972025-01-10T16:37:23.155Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.206719281Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"client":"2","":"downstairs","session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2982025-01-10T16:37:23.155Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.20673242Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
2992025-01-10T16:37:23.155Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.20674416Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
3002025-01-10T16:37:23.155Z{"msg":"49b48452-2fc8-405e-8753-275c5f5f6a7e is now active with session: c2af369d-f397-4ef7-9bd4-3b9e524320fb","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.206763419Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
3012025-01-10T16:37:23.155Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.206775428Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227,"session_id":"c2af369d-f397-4ef7-9bd4-3b9e524320fb"}
3022025-01-10T16:37:23.155Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:22.206787198Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227}
3032025-01-10T16:37:23.155ZDisk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
3042025-01-10T16:37:23.161ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
3052025-01-10T16:37:23.161ZFill test
3062025-01-10T16:37:23.161ZRead and Verify all blocks (0..50 range:false)
3072025-01-10T16:37:23.164ZWrote out file "/var/tmp/test_up-build/verify_file"
3082025-01-10T16:37:23.164ZCLIENT: Tests done. All submitted work has been ACK'd
3092025-01-10T16:37:23.164Z----------------------------------------------------------------
3102025-01-10T16:37:23.164Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
3112025-01-10T16:37:23.164ZStates: Active Active Active
3122025-01-10T16:37:23.164ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
3132025-01-10T16:37:23.164Z 1002 Acked Read 50 Done Done Done false
3142025-01-10T16:37:23.164Z STATES DS:0 DS:1 DS:2 TOTAL
3152025-01-10T16:37:23.164Z Sent 0 0 0 0
3162025-01-10T16:37:23.164Z Done 1 1 1 3
3172025-01-10T16:37:23.164Z Skipped 0 0 0 0
3182025-01-10T16:37:23.164Z Error 0 0 0 0
3192025-01-10T16:37:23.164ZLast Flush: 1001 1001 1001
3202025-01-10T16:37:23.164ZDownstairs last five completed: 1001 1000
3212025-01-10T16:37:23.164ZUpstairs last five completed: 1002 1001 1000
3222025-01-10T16:37:23.164ZCLIENT: Up:0 ds:1 act:3
3232025-01-10T16:37:27.165Z----------------------------------------------------------------
3242025-01-10T16:37:27.165Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
3252025-01-10T16:37:27.165ZDownstairs last five completed: 1003 1002 1001 1000
3262025-01-10T16:37:27.165ZUpstairs last five completed: 1003 1002 1001 1000
3272025-01-10T16:37:27.165ZCLIENT: Up:0 ds:0 act:3
3282025-01-10T16:37:27.165ZCLIENT: All crucible jobs finished, exiting program
3292025-01-10T16:37:27.168Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-10T16:37:26.219623453Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1227}
3302025-01-10T16:37:27.173ZRepair part 1 passed
3312025-01-10T16:37:27.175Z
3322025-01-10T16:37:27.175ZKill the current downstairs
3332025-01-10T16:37:27.345ZDownstairs 2 stopped
3342025-01-10T16:37:27.345Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
3352025-01-10T16:37:27.348ZNow put back the original so we have a mismatch
3362025-01-10T16:37:27.348Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
3372025-01-10T16:37:27.350ZRestart downstairs with old directory
3382025-01-10T16:37:27.406Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
3392025-01-10T16:37:27.423ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3402025-01-10T16:37:27.425Z 0 000-009 32 32 31 1498 1498 1497 F F F <---
3412025-01-10T16:37:27.425Z 1 010-019 32 32 31 1498 1498 1497 F F F <---
3422025-01-10T16:37:27.425Z 2 020-029 32 32 31 1498 1498 1497 F F F <---
3432025-01-10T16:37:27.425Z 3 030-039 32 32 31 1498 1498 1497 F F F <---
3442025-01-10T16:37:27.425Z 4 040-049 32 32 31 1498 1498 1497 F F F <---
3452025-01-10T16:37:27.425ZMax gen: 32, Max flush: 1498
3462025-01-10T16:37:27.425ZError: Difference in extent metadata found!
3472025-01-10T16:37:27.428Zdump test found error as expected
3482025-01-10T16:37:27.428Z
3492025-01-10T16:37:27.428Z
3502025-01-10T16:37:27.428Z/var/tmp/bins/crutest verify --range -g 33 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
3512025-01-10T16:37:27.485Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.539329256Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"task":"crutest"}
3522025-01-10T16:37:27.488Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.539899383Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"task":"crutest"}
3532025-01-10T16:37:27.488Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.541563527Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"task":"crutest"}
3542025-01-10T16:37:27.488Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.54174356Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235}
3552025-01-10T16:37:27.488Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56068760771d58f96ac946e6b2ae83073c18c1b4\",\n git_commit_timestamp: \"2025-01-10T16:19:36.000000000Z\",\n git_branch: \"alan/rust-1-84-please\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.541780628Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235}
3562025-01-10T16:37:27.488Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.541795667Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235}
3572025-01-10T16:37:27.488Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.541809807Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235}
3582025-01-10T16:37:27.493Z{"msg":"Crucible 85c1dd59-3f60-4184-bf0d-d8509843d983 has session id: 2a0472c3-83ab-4b2c-83bc-3563e281bf2b","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.542466031Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3592025-01-10T16:37:27.493Z{"msg":"Upstairs opts: Upstairs UUID: 85c1dd59-3f60-4184-bf0d-d8509843d983, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.542532088Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3602025-01-10T16:37:27.493Z{"msg":"Crucible stats registered with UUID: 85c1dd59-3f60-4184-bf0d-d8509843d983","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.542561577Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3612025-01-10T16:37:27.524Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.578074643Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"task":"crutest"}
3622025-01-10T16:37:27.524Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.578132181Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235}
3632025-01-10T16:37:27.524Z{"msg":"85c1dd59-3f60-4184-bf0d-d8509843d983 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.57814933Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3642025-01-10T16:37:27.524Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.578172519Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"client":"0","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3652025-01-10T16:37:27.527Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.578223057Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"client":"0","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3662025-01-10T16:37:27.527Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.578255596Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"client":"1","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3672025-01-10T16:37:27.527Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.578285265Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"client":"1","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3682025-01-10T16:37:27.527Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.578312964Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"client":"2","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3692025-01-10T16:37:27.527Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.578343242Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"client":"2","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3702025-01-10T16:37:27.527Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.578373591Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"":"io task","client":"0","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3712025-01-10T16:37:27.527Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.578417269Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"":"io task","client":"1","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3722025-01-10T16:37:27.527Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.578457248Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3732025-01-10T16:37:27.527Z{"msg":"ds_connection connected from Ok(127.0.0.1:45369)","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.578485167Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"":"io task","client":"1","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3742025-01-10T16:37:27.527Z{"msg":"ds_connection connected from Ok(127.0.0.1:34812)","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.578512246Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"":"io task","client":"0","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3752025-01-10T16:37:27.527Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-01-10T16:37:26.578557914Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3762025-01-10T16:37:27.527Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-01-10T16:37:26.57866362Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3772025-01-10T16:37:27.527Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-01-10T16:37:26.578718888Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3782025-01-10T16:37:27.527Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.578870351Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3792025-01-10T16:37:27.527Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.579396381Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"client":"0","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3802025-01-10T16:37:27.527Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.579441939Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"client":"0","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3812025-01-10T16:37:27.527Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.579470027Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"client":"1","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3822025-01-10T16:37:27.527Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.579505846Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"client":"1","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3832025-01-10T16:37:27.527Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.579961818Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"client":"1","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3842025-01-10T16:37:27.527Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.580107382Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"client":"0","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3852025-01-10T16:37:27.527Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.580262376Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"client":"1","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3862025-01-10T16:37:27.527Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.580297065Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"client":"1","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3872025-01-10T16:37:27.527Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.580312144Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3882025-01-10T16:37:27.527Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.580436919Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"client":"0","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3892025-01-10T16:37:27.527Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.580506476Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"client":"0","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3902025-01-10T16:37:27.527Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:26.580543855Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3912025-01-10T16:37:37.526Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.57920278Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3922025-01-10T16:37:37.526Z{"msg":"ds_connection connected from Ok(127.0.0.1:42611)","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.579292556Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3932025-01-10T16:37:37.530Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.580410742Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"client":"2","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3942025-01-10T16:37:37.530Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.5804582Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"client":"2","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3952025-01-10T16:37:37.530Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.580974939Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"client":"2","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3962025-01-10T16:37:37.530Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.581306936Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"client":"2","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3972025-01-10T16:37:37.530Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.581335775Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"client":"2","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3982025-01-10T16:37:37.530Z{"msg":"[0]R flush_numbers: [1498, 1498, 1498, 1498, 1498]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.581350884Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
3992025-01-10T16:37:37.530Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.581365614Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4002025-01-10T16:37:37.531Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.581385903Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4012025-01-10T16:37:37.531Z{"msg":"[1]R flush_numbers: [1498, 1498, 1498, 1498, 1498]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.581399822Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4022025-01-10T16:37:37.531Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.581412952Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4032025-01-10T16:37:37.531Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.581426841Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4042025-01-10T16:37:37.531Z{"msg":"[2]R flush_numbers: [1497, 1497, 1497, 1497, 1497]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.5814469Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4052025-01-10T16:37:37.531Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.58146025Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4062025-01-10T16:37:37.531Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.581474329Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4072025-01-10T16:37:37.531Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.581494189Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4082025-01-10T16:37:37.531Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.581507278Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4092025-01-10T16:37:37.531Z{"msg":"Next flush: 1499","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.581526717Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4102025-01-10T16:37:37.531Z{"msg":"Extent 0 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.581539797Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4112025-01-10T16:37:37.531Z{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.581560386Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4122025-01-10T16:37:37.531Z{"msg":"extent:0 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.581574795Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4132025-01-10T16:37:37.531Z{"msg":"extent:0 flush: 1498 1498 1497 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.581596614Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4142025-01-10T16:37:37.531Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.581615504Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4152025-01-10T16:37:37.531Z{"msg":"extent:0 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.581634903Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4162025-01-10T16:37:37.531Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.581649482Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4172025-01-10T16:37:37.531Z{"msg":"find dest for source 0 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.581677721Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4182025-01-10T16:37:37.531Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.581692321Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4192025-01-10T16:37:37.531Z{"msg":"Extent 1 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.581721229Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4202025-01-10T16:37:37.531Z{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.581743669Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4212025-01-10T16:37:37.531Z{"msg":"extent:1 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.581757738Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4222025-01-10T16:37:37.531Z{"msg":"extent:1 flush: 1498 1498 1497 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.581778567Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4232025-01-10T16:37:37.531Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.581792557Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4242025-01-10T16:37:37.531Z{"msg":"extent:1 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.581817506Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4252025-01-10T16:37:37.531Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.581843495Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4262025-01-10T16:37:37.531Z{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.581891553Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4272025-01-10T16:37:37.531Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.581926391Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4282025-01-10T16:37:37.531Z{"msg":"Extent 2 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.58196026Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4292025-01-10T16:37:37.531Z{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.581984749Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4302025-01-10T16:37:37.531Z{"msg":"extent:2 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.582015858Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4312025-01-10T16:37:37.531Z{"msg":"extent:2 flush: 1498 1498 1497 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.582041787Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4322025-01-10T16:37:37.531Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.582075275Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4332025-01-10T16:37:37.531Z{"msg":"extent:2 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.582100385Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4342025-01-10T16:37:37.531Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.582128893Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4352025-01-10T16:37:37.531Z{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.582153652Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4362025-01-10T16:37:37.531Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.582187031Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4372025-01-10T16:37:37.531Z{"msg":"Extent 3 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.58221233Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4382025-01-10T16:37:37.532Z{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.582249608Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4392025-01-10T16:37:37.532Z{"msg":"extent:3 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.582279847Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4402025-01-10T16:37:37.532Z{"msg":"extent:3 flush: 1498 1498 1497 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.582324176Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4412025-01-10T16:37:37.532Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.582369484Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4422025-01-10T16:37:37.532Z{"msg":"extent:3 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.582408642Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4432025-01-10T16:37:37.535Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.582434321Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4442025-01-10T16:37:37.535Z{"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.58246953Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4452025-01-10T16:37:37.535Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.582496029Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4462025-01-10T16:37:37.535Z{"msg":"Extent 4 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.582527928Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4472025-01-10T16:37:37.535Z{"msg":"First source client ID for extent 4","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.582552656Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4482025-01-10T16:37:37.535Z{"msg":"extent:4 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.582578965Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4492025-01-10T16:37:37.535Z{"msg":"extent:4 flush: 1498 1498 1497 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.582612794Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4502025-01-10T16:37:37.535Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.582638373Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4512025-01-10T16:37:37.535Z{"msg":"extent:4 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.582671202Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4522025-01-10T16:37:37.535Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.582688491Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4532025-01-10T16:37:37.535Z{"msg":"find dest for source 0 for extent at index 4","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.58271007Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4542025-01-10T16:37:37.535Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.582735259Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4552025-01-10T16:37:37.535Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.582758778Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"client":"0","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4562025-01-10T16:37:37.535Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.582772358Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"client":"1","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4572025-01-10T16:37:37.535Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.582794457Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"client":"2","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4582025-01-10T16:37:37.535Z{"msg":"Full repair list: {ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.582812266Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4592025-01-10T16:37:37.535Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 1499, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 1499, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 1499, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 1499, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 1499, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.582895303Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4602025-01-10T16:37:37.535Z{"msg":"starting reconciliation 510dcdcc-b492-4dcb-9105-5417c7048b30: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.582923232Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4612025-01-10T16:37:37.535Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-10T16:37:36.582940071Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"reconcile":"510dcdcc-b492-4dcb-9105-5417c7048b30","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4622025-01-10T16:37:37.535Z{"msg":"no Nexus client from DNS, aborting start notification","v":0,"name":"crucible","level":50,"time":"2025-01-10T16:37:36.58295577Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"reconcile":"510dcdcc-b492-4dcb-9105-5417c7048b30","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4632025-01-10T16:37:37.535Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-10T16:37:36.58297785Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"reconcile":"510dcdcc-b492-4dcb-9105-5417c7048b30","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4642025-01-10T16:37:37.535Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.582992679Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"client":"2","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4652025-01-10T16:37:37.536Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-10T16:37:36.583015208Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"reconcile":"510dcdcc-b492-4dcb-9105-5417c7048b30","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4662025-01-10T16:37:37.539Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-10T16:37:36.590138494Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"reconcile":"510dcdcc-b492-4dcb-9105-5417c7048b30","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4672025-01-10T16:37:37.539Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-10T16:37:36.590555898Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"reconcile":"510dcdcc-b492-4dcb-9105-5417c7048b30","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4682025-01-10T16:37:37.539Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-10T16:37:36.590825907Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"reconcile":"510dcdcc-b492-4dcb-9105-5417c7048b30","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4692025-01-10T16:37:37.539Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.591120985Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"client":"2","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4702025-01-10T16:37:37.539Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-10T16:37:36.591163154Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"reconcile":"510dcdcc-b492-4dcb-9105-5417c7048b30","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4712025-01-10T16:37:37.541Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-10T16:37:36.593974142Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"reconcile":"510dcdcc-b492-4dcb-9105-5417c7048b30","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4722025-01-10T16:37:37.541Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-10T16:37:36.594368616Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"reconcile":"510dcdcc-b492-4dcb-9105-5417c7048b30","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4732025-01-10T16:37:37.541Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-10T16:37:36.594599337Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"reconcile":"510dcdcc-b492-4dcb-9105-5417c7048b30","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4742025-01-10T16:37:37.544Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.594939513Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"client":"2","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4752025-01-10T16:37:37.544Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-10T16:37:36.594988811Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"reconcile":"510dcdcc-b492-4dcb-9105-5417c7048b30","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4762025-01-10T16:37:37.547Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-10T16:37:36.597670195Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"reconcile":"510dcdcc-b492-4dcb-9105-5417c7048b30","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4772025-01-10T16:37:37.547Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-10T16:37:36.598052059Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"reconcile":"510dcdcc-b492-4dcb-9105-5417c7048b30","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4782025-01-10T16:37:37.547Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-10T16:37:36.598312539Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"reconcile":"510dcdcc-b492-4dcb-9105-5417c7048b30","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4792025-01-10T16:37:37.547Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.598598808Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"client":"2","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4802025-01-10T16:37:37.547Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-10T16:37:36.598645646Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"reconcile":"510dcdcc-b492-4dcb-9105-5417c7048b30","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4812025-01-10T16:37:37.550Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-10T16:37:36.601330989Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"reconcile":"510dcdcc-b492-4dcb-9105-5417c7048b30","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4822025-01-10T16:37:37.550Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-10T16:37:36.601718784Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"reconcile":"510dcdcc-b492-4dcb-9105-5417c7048b30","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4832025-01-10T16:37:37.550Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-10T16:37:36.601976173Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"reconcile":"510dcdcc-b492-4dcb-9105-5417c7048b30","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4842025-01-10T16:37:37.550Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.60230137Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"client":"2","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4852025-01-10T16:37:37.550Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-10T16:37:36.602343299Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"reconcile":"510dcdcc-b492-4dcb-9105-5417c7048b30","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4862025-01-10T16:37:37.553Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2025-01-10T16:37:36.604939385Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"reconcile":"510dcdcc-b492-4dcb-9105-5417c7048b30","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4872025-01-10T16:37:37.553Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.60532022Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4882025-01-10T16:37:37.553Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.605358128Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"client":"0","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4892025-01-10T16:37:37.553Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.605383358Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"client":"1","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4902025-01-10T16:37:37.553Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.605416056Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235Wait for a query_work_queue command to finish before sending IO
4912025-01-10T16:37:37.553Z,"client":"2","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4922025-01-10T16:37:37.553Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.605444205Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4932025-01-10T16:37:37.553Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.605470884Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4942025-01-10T16:37:37.553Z{"msg":"85c1dd59-3f60-4184-bf0d-d8509843d983 is now active with session: 2a0472c3-83ab-4b2c-83bc-3563e281bf2b","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.605494693Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4952025-01-10T16:37:37.553Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-01-10T16:37:36.605520952Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235}
4962025-01-10T16:37:37.553Z{"msg":"no Nexus client from DNS, aborting finish notification","v":0,"name":"crucible","level":50,"time":"2025-01-10T16:37:36.605540691Z","hostname":"w-01JH8H1HV4PDNPGET5JKDVHX88","pid":1235,"reconcile":"510dcdcc-b492-4dcb-9105-5417c7048b30","":"downstairs","":"downstairs","session_id":"2a0472c3-83ab-4b2c-83bc-3563e281bf2b"}
4972025-01-10T16:37:37.553ZDisk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
4982025-01-10T16:37:37.553ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4992025-01-10T16:37:37.553ZRead and Verify all blocks (0..50 range:true)
5002025-01-10T16:37:37.558ZWrote out file "/var/tmp/test_up-build/verify_file"
5012025-01-10T16:37:37.558ZVerify test completed
5022025-01-10T16:37:37.561ZWrote out file "/var/tmp/test_up-build/verify_file"
5032025-01-10T16:37:37.561ZCLIENT: Tests done. All submitted work has been ACK'd
5042025-01-10T16:37:37.561Z----------------------------------------------------------------
5052025-01-10T16:37:37.561Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
5062025-01-10T16:37:37.561ZStates: Active Active Active
5072025-01-10T16:37:37.561ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
5082025-01-10T16:37:37.561Z 1000 Acked Read 50 Done Done Done false
5092025-01-10T16:37:37.561Z STATES DS:0 DS:1 DS:2 TOTAL
5102025-01-10T16:37:37.561Z Sent 0 0 0 0
5112025-01-10T16:37:37.561Z Done 1 1 1 3
5122025-01-10T16:37:37.561Z Skipped 0 0 0 0
5132025-01-10T16:37:37.561Z Error 0 0 0 0
5142025-01-10T16:37:37.561ZLast Flush: 0 0 0
5152025-01-10T16:37:37.561ZDownstairs last five completed:
5162025-01-10T16:37:37.561ZUpstairs last five completed: 1000
5172025-01-10T16:37:37.561ZCLIENT: Up:0 ds:1 act:3
5182025-01-10T16:37:41.561Z----------------------------------------------------------------
5192025-01-10T16:37:41.561Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
5202025-01-10T16:37:41.561ZDownstairs last five completed: 1001 1000
5212025-01-10T16:37:41.561ZUpstairs last five completed: 1001 1000
5222025-01-10T16:37:41.561ZCLIENT: Up:0 ds:0 act:3
5232025-01-10T16:37:41.561ZCLIENT: All crucible jobs finished, exiting program
5242025-01-10T16:37:41.568ZRepair part 2 passed
5252025-01-10T16:37:41.571Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
5262025-01-10T16:37:41.586ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
5272025-01-10T16:37:41.586Z 0 000-009 32 32 32 1498 1498 1498 F F F
5282025-01-10T16:37:41.586Z 1 010-019 32 32 32 1498 1498 1498 F F F
5292025-01-10T16:37:41.586Z 2 020-029 32 32 32 1498 1498 1498 F F F
5302025-01-10T16:37:41.586Z 3 030-039 32 32 32 1498 1498 1498 F F F
5312025-01-10T16:37:41.586Z 4 040-049 32 32 32 1498 1498 1498 F F F
5322025-01-10T16:37:41.589ZMax gen: 32, Max flush: 1498
5332025-01-10T16:37:41.593Zdump test passed
5342025-01-10T16:37:41.593Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
5352025-01-10T16:37:41.608Z Extent 1
5362025-01-10T16:37:41.617ZGEN 32 32 32
5372025-01-10T16:37:41.617ZFLUSH_ID 1498 1498 1498
5382025-01-10T16:37:41.617ZDIRTY
5392025-01-10T16:37:41.617Z
5402025-01-10T16:37:41.617ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
5412025-01-10T16:37:41.617Z010 A A A A A A
5422025-01-10T16:37:41.617Z011 A A A A A A
5432025-01-10T16:37:41.617Z012 A A A A A A
5442025-01-10T16:37:41.617Z013 A A A A A A
5452025-01-10T16:37:41.618Z014 A A A A A A
5462025-01-10T16:37:41.618Z015 A A A A A A
5472025-01-10T16:37:41.627Z016 A A A A A A
5482025-01-10T16:37:41.627Z017 A A A A A A
5492025-01-10T16:37:41.627Z018 A A A A A A
5502025-01-10T16:37:41.628Z019 A A A A A A
5512025-01-10T16:37:41.628Zdump extent test passed
5522025-01-10T16:37:41.628Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
5532025-01-10T16:37:41.643ZExtent 2 Block in extent 0 Actual block 20
5542025-01-10T16:37:41.643Z
5552025-01-10T16:37:41.645Z DATA SHA256 VER
5562025-01-10T16:37:41.645Z------ ---------------------------------------------------------------- ---
5572025-01-10T16:37:41.648Z 0 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
5582025-01-10T16:37:41.648Z 1 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
5592025-01-10T16:37:41.648Z 2 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
5602025-01-10T16:37:41.648Z
5612025-01-10T16:37:41.648ZNONCES 0 1 2 DIFF
5622025-01-10T16:37:41.648Z------ ------------------------ ------------------------ ------------------------ -----
5632025-01-10T16:37:41.648Z
5642025-01-10T16:37:41.648Z TAGS 0 1 2 DIFF
5652025-01-10T16:37:41.648Z------ -------------------------------- -------------------------------- -------------------------------- -----
5662025-01-10T16:37:41.648Z
5672025-01-10T16:37:41.648ZHASHES 0 1 2 DIFF
5682025-01-10T16:37:41.648Z------ ---------------- ---------------- ---------------- -----
5692025-01-10T16:37:41.648Z 0 42916ab6fa364c70 42916ab6fa364c70 42916ab6fa364c70
5702025-01-10T16:37:41.648Z
5712025-01-10T16:37:41.650Zdump block test passed
5722025-01-10T16:37:41.651ZInitial upstairs tests have completed, stopping all downstairs
5732025-01-10T16:37:46.714ZCreating 4 larger downstairs regions
5742025-01-10T16:37:46.830ZStarting 4 downstairs
5752025-01-10T16:37:51.837Zdsc restarted at PID: 1248
5762025-01-10T16:37:52.067ZNow do the replace-reconcile test
5772025-01-10T16:37:52.125ZUsing 8840 for the replacement port
5782025-01-10T16:41:04.661ZCompleted test: replace-reconcile
5792025-01-10T16:41:04.661ZNow do the replace-before-active test
5802025-01-10T16:43:49.056ZCompleted test: replace-before-active
5812025-01-10T16:43:49.056ZAll tests have completed, stopping all downstairs
5822025-01-10T16:43:49.116Z
5832025-01-10T16:43:49.116ZAll Tests have passed
5842025-01-10T16:43:49.118Z8:34 Test duration
5852025-01-10T16:43:49.118Z
5862025-01-10T16:43:49.118Zreal 8:33.954806181
5872025-01-10T16:43:49.118Zuser 1:04.207088339
5882025-01-10T16:43:49.118Zsys 10.201858840
5892025-01-10T16:43:49.118Ztrap 0.088354181
5902025-01-10T16:43:49.118Ztflt 0.013837757
5912025-01-10T16:43:49.118Zdflt 0.014653909
5922025-01-10T16:43:49.118Zkflt 0.001094933
5932025-01-10T16:43:49.118Zlock 6:05:22.427433064
5942025-01-10T16:43:49.118Zslp 26:17.391295100
5952025-01-10T16:43:49.118Zlat 8.132342067
5962025-01-10T16:43:49.118Zstop 0.132901384
5972025-01-10T16:43:49.118Z+ echo 'test-up-unencrypted ends'
5982025-01-10T16:43:49.119Ztest-up-unencrypted ends
5992025-01-10T16:43:54.123Zprocess exited: duration 522117 ms, exit code 0
6002025-01-10T16:43:54.123Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
6012025-01-10T16:44:54.166Zexec warning: : stdout descriptor may be held open by a background process; giving up!
6022025-01-10T16:44:54.166Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
6032025-01-10T16:44:54.180Zfound 14 output files
6042025-01-10T16:44:54.180Zuploading: /tmp/test_up-build/dsc-out.txt (42133 bytes)
6052025-01-10T16:44:55.195Zuploaded: /tmp/test_up-build/dsc-out.txt
6062025-01-10T16:44:55.201Zuploading: /tmp/test_up-build/test_up_out.txt (10791844 bytes)
6072025-01-10T16:44:56.361Zuploaded: /tmp/test_up-build/test_up_out.txt
6082025-01-10T16:44:56.364Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (2495 bytes)
6092025-01-10T16:44:57.369Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
6102025-01-10T16:44:57.372Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (144631 bytes)
6112025-01-10T16:44:58.381Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
6122025-01-10T16:44:58.384Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (251001 bytes)
6132025-01-10T16:44:59.395Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
6142025-01-10T16:44:59.397Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (139425 bytes)
6152025-01-10T16:45:00.406Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
6162025-01-10T16:45:00.409Zuploading: /tmp/debug/df.txt (1270 bytes)
6172025-01-10T16:45:01.415Zuploaded: /tmp/debug/df.txt
6182025-01-10T16:45:01.418Zuploading: /tmp/debug/dtrace.txt (485432 bytes)
6192025-01-10T16:45:02.433Zuploaded: /tmp/debug/dtrace.txt
6202025-01-10T16:45:02.435Zuploading: /tmp/debug/iostat.txt (174012 bytes)
6212025-01-10T16:45:02.441Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 174012 -> 176412
6222025-01-10T16:45:03.447Zuploaded: /tmp/debug/iostat.txt
6232025-01-10T16:45:03.450Zuploading: /tmp/debug/mpstat.txt (422825 bytes)
6242025-01-10T16:45:03.460Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 422825 -> 429386
6252025-01-10T16:45:04.466Zuploaded: /tmp/debug/mpstat.txt
6262025-01-10T16:45:04.469Zuploading: /tmp/debug/paging.txt (75864 bytes)
6272025-01-10T16:45:04.473Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 75864 -> 77172
6282025-01-10T16:45:05.478Zuploaded: /tmp/debug/paging.txt
6292025-01-10T16:45:05.481Zuploading: /tmp/debug/prstat.txt (794467 bytes)
6302025-01-10T16:45:05.496Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 794467 -> 809997
6312025-01-10T16:45:06.501Zuploaded: /tmp/debug/prstat.txt
6322025-01-10T16:45:06.504Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
6332025-01-10T16:45:07.510Zuploaded: /tmp/debug/psrinfo.txt
6342025-01-10T16:45:07.513Zuploading: /tmp/debug/upstairs-info.txt (80186 bytes)
6352025-01-10T16:45:08.521Zuploaded: /tmp/debug/upstairs-info.txt