01JE8WYCG0DTMD0EF3K2BMDHED: test-up-unencrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01JE8WYND2VXAEDBVAE0E8M2ZB

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-12-04T13:29:16.894Zjob dependencies complete; ready to run (waiting for 14 m 12 s)
22024-12-04T13:30:22.070Zjob assigned to worker 01JE8XRS07NFFENGZRQFRBMQZH [factory aws, i-0b574bf79d930313e] (queued for 1 m 5 s)
32024-12-04T13:30:30.211Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-12-04T13:30:31.308Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-12-04T13:30:31.308Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-12-04T13:30:32.363Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-12-04T13:30:32.363Zdownloading input: /input/build/work/bins/crutest.gz
82024-12-04T13:30:33.462Zdownloaded input: /input/build/work/bins/crutest.gz
92024-12-04T13:30:33.462Zdownloading input: /input/build/work/bins/dsc.gz
102024-12-04T13:30:34.134Zdownloaded input: /input/build/work/bins/dsc.gz
112024-12-04T13:30:34.134Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-12-04T13:30:34.151Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-12-04T13:30:34.151Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-12-04T13:30:34.168Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-12-04T13:30:34.168Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-12-04T13:30:34.184Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-12-04T13:30:34.184Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-12-04T13:30:34.203Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-12-04T13:30:34.203Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-12-04T13:30:34.221Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-12-04T13:30:34.222Zdownloading input: /input/build/work/scripts/test_up.sh
222024-12-04T13:30:34.238Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-12-04T13:30:34.238Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-12-04T13:30:34.257Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-12-04T13:30:34.259Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-12-04T13:30:34.979Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-12-04T13:30:34.980Zstarting task 0: "setup"
282024-12-04T13:30:34.989Z++ uname -s
292024-12-04T13:30:34.992Z+ kern=SunOS
302024-12-04T13:30:34.992Z+ case "$kern" in
312024-12-04T13:30:34.992Z+ groupadd -g 12345 build
322024-12-04T13:30:34.994Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-12-04T13:30:37.000Z+ zfs create -o mountpoint=/work rpool/work
342024-12-04T13:30:37.065Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-12-04T13:30:37.069Z+ home_fs=zfs
362024-12-04T13:30:37.069Z+ [[ zfs == autofs ]]
372024-12-04T13:30:37.069Z+ mkdir -p /home/build
382024-12-04T13:30:37.069Z+ chown build:build /home/build /work
392024-12-04T13:30:39.071Z+ chmod 0700 /home/build /work
402024-12-04T13:30:39.073Zprocess exited: duration 4095 ms, exit code 0
 
412024-12-04T13:30:39.078Zstarting task 1: "authentication"
422024-12-04T13:30:39.095Zprocess exited: duration 16 ms, exit code 0
 
432024-12-04T13:30:39.102Zstarting task 2: "build"
442024-12-04T13:30:39.105Z+ banner cores
452024-12-04T13:30:39.107Z
462024-12-04T13:30:39.107Z #### #### ##### ###### ####
472024-12-04T13:30:39.107Z # # # # # # # #
482024-12-04T13:30:39.107Z # # # # # ##### ####
492024-12-04T13:30:39.108Z # # # ##### # #
502024-12-04T13:30:39.108Z # # # # # # # # #
512024-12-04T13:30:39.108Z #### #### # # ###### ####
522024-12-04T13:30:39.108Z
532024-12-04T13:30:39.110Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-12-04T13:30:39.116Z+ echo 'input bins dir contains:'
552024-12-04T13:30:39.116Zinput bins dir contains:
562024-12-04T13:30:39.116Z+ ls -ltr /input/build/work/bins
572024-12-04T13:30:39.118Ztotal 761866
582024-12-04T13:30:39.118Z-rw-r--r-- 1 root root 109324428 Dec 4 13:30 crucible-downstairs.gz
592024-12-04T13:30:39.119Z-rw-r--r-- 1 root root 104619017 Dec 4 13:30 crucible-hammer.gz
602024-12-04T13:30:39.119Z-rw-r--r-- 1 root root 109319980 Dec 4 13:30 crutest.gz
612024-12-04T13:30:39.119Z-rw-r--r-- 1 root root 66562355 Dec 4 13:30 dsc.gz
622024-12-04T13:30:39.119Z+ banner unpack
632024-12-04T13:30:39.121Z
642024-12-04T13:30:39.121Z # # # # ##### ## #### # #
652024-12-04T13:30:39.121Z # # ## # # # # # # # # #
662024-12-04T13:30:39.121Z # # # # # # # # # # ####
672024-12-04T13:30:39.121Z # # # # # ##### ###### # # #
682024-12-04T13:30:39.121Z # # # ## # # # # # # #
692024-12-04T13:30:39.121Z #### # # # # # #### # #
702024-12-04T13:30:39.121Z
712024-12-04T13:30:39.121Z+ mkdir -p /var/tmp/bins
722024-12-04T13:30:39.124Z+ for t in "$input/bins/"*.gz
732024-12-04T13:30:39.124Z++ basename /input/build/work/bins/crucible-downstairs.gz
742024-12-04T13:30:39.124Z+ b=crucible-downstairs.gz
752024-12-04T13:30:39.124Z+ b=crucible-downstairs
762024-12-04T13:30:39.124Z+ gunzip
772024-12-04T13:30:41.420Z+ chmod +x /var/tmp/bins/crucible-downstairs
782024-12-04T13:30:41.422Z+ for t in "$input/bins/"*.gz
792024-12-04T13:30:41.427Z++ basename /input/build/work/bins/crucible-hammer.gz
802024-12-04T13:30:41.427Z+ b=crucible-hammer.gz
812024-12-04T13:30:41.427Z+ b=crucible-hammer
822024-12-04T13:30:41.427Z+ gunzip
832024-12-04T13:30:43.595Z+ chmod +x /var/tmp/bins/crucible-hammer
842024-12-04T13:30:43.597Z+ for t in "$input/bins/"*.gz
852024-12-04T13:30:43.600Z++ basename /input/build/work/bins/crutest.gz
862024-12-04T13:30:43.600Z+ b=crutest.gz
872024-12-04T13:30:43.600Z+ b=crutest
882024-12-04T13:30:43.600Z+ gunzip
892024-12-04T13:30:45.918Z+ chmod +x /var/tmp/bins/crutest
902024-12-04T13:30:45.921Z+ for t in "$input/bins/"*.gz
912024-12-04T13:30:45.923Z++ basename /input/build/work/bins/dsc.gz
922024-12-04T13:30:45.923Z+ b=dsc.gz
932024-12-04T13:30:45.923Z+ b=dsc
942024-12-04T13:30:45.923Z+ gunzip
952024-12-04T13:30:47.311Z+ chmod +x /var/tmp/bins/dsc
962024-12-04T13:30:47.314Z+ export BINDIR=/var/tmp/bins
972024-12-04T13:30:47.314Z+ BINDIR=/var/tmp/bins
982024-12-04T13:30:47.314Z+ jobpid=1144
992024-12-04T13:30:47.317Z+ echo 'Setup debug logging'
1002024-12-04T13:30:47.317ZSetup debug logging
1012024-12-04T13:30:47.317Z+ mkdir /tmp/debug
1022024-12-04T13:30:47.317Z+ sleep 7200
1032024-12-04T13:30:47.317Z+ psrinfo -v
1042024-12-04T13:30:47.320Z+ df -h
1052024-12-04T13:30:47.327Z+ prstat -d d -mLc 1
1062024-12-04T13:30:47.329Z+ iostat -T d -xn 1
1072024-12-04T13:30:47.329Z+ mpstat -T d 1
1082024-12-04T13:30:47.329Z+ vmstat -T d -p 1
1092024-12-04T13:30:47.331Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1102024-12-04T13:30:47.332Z+ banner test_up_unencrypted
1112024-12-04T13:30:47.332Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1122024-12-04T13:30:47.334Z
1132024-12-04T13:30:47.334Z ##### ###### #### ##### # # ##### # # # #
1142024-12-04T13:30:47.334Z # # # # # # # # # # ## #
1152024-12-04T13:30:47.334Z # ##### #### # # # # # # # # # #
1162024-12-04T13:30:47.334Z # # # # # # ##### # # # # #
1172024-12-04T13:30:47.334Z # # # # # # # # # # # ##
1182024-12-04T13:30:47.334Z # ###### #### # ####### #### # ####### #### # #
1192024-12-04T13:30:47.334Z
1202024-12-04T13:30:47.334Z+ ptime -m bash /input/build/work/scripts/test_up.sh -N unencrypted
1212024-12-04T13:30:47.344Z/input/build/work
1222024-12-04T13:30:47.355ZTurn off color for downstairs dump
1232024-12-04T13:30:47.357Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1242024-12-04T13:30:47.357ZCreating 3 downstairs regions
1252024-12-04T13:30:47.676ZStarting 3 downstairs
1262024-12-04T13:30:47.679Zdsc started at PID: 1189
1272024-12-04T13:30:52.857ZDisable automatic restart on all downstairs
1282024-12-04T13:30:52.910Z
1292024-12-04T13:30:52.910ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1302024-12-04T13:30:52.914ZRunning test: span
1312024-12-04T13:30:57.054ZCompleted test: span
1322024-12-04T13:31:02.061ZRunning test: big
1332024-12-04T13:31:06.241ZCompleted test: big
1342024-12-04T13:31:11.248ZRunning test: dep
1352024-12-04T13:31:23.938ZCompleted test: dep
1362024-12-04T13:31:28.945ZRunning test: balloon
1372024-12-04T13:31:38.544ZCompleted test: balloon
1382024-12-04T13:31:43.551ZRunning test: deactivate
1392024-12-04T13:32:37.799ZCompleted test: deactivate
1402024-12-04T13:32:42.807ZRunning hammer
1412024-12-04T13:32:45.935ZRun repair tests
1422024-12-04T13:32:45.939Z/var/tmp/bins/crutest fill -g 31 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1432024-12-04T13:32:45.997Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.230778767Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"task":"crutest"}
1442024-12-04T13:32:45.999Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.231349457Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"task":"crutest"}
1452024-12-04T13:32:45.999Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.233097415Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"task":"crutest"}
1462024-12-04T13:32:45.999Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.233283385Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226}
1472024-12-04T13:32:45.999Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0e1f7f60aff9f26b0d737c97120445edbded1ad3\",\n git_commit_timestamp: \"2024-12-04T13:14:36.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.233323941Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226}
1482024-12-04T13:32:45.999Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.233339849Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226}
1492024-12-04T13:32:45.999Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.233363587Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226}
1502024-12-04T13:32:46.002Z{"msg":"Crucible f486fb3c-423f-4cd3-ac15-e0c5f051c80e has session id: e5f0862b-402f-4f0e-8b96-27a77224129d","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.233735068Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1512024-12-04T13:32:46.002Z{"msg":"Upstairs opts: Upstairs UUID: f486fb3c-423f-4cd3-ac15-e0c5f051c80e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.233788893Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1522024-12-04T13:32:46.002Z{"msg":"Crucible stats registered with UUID: f486fb3c-423f-4cd3-ac15-e0c5f051c80e","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.233834338Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1532024-12-04T13:32:46.032Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.266958346Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"task":"crutest"}
1542024-12-04T13:32:46.032Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.267018659Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226}
1552024-12-04T13:32:46.033Z{"msg":"f486fb3c-423f-4cd3-ac15-e0c5f051c80e active request set","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.267039197Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1562024-12-04T13:32:46.036Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.267063965Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"client":"0","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1572024-12-04T13:32:46.036Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.267102581Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"client":"0","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1582024-12-04T13:32:46.036Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.267155755Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"client":"1","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1592024-12-04T13:32:46.036Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.267186362Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"client":"1","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1602024-12-04T13:32:46.036Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.26720632Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"client":"2","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1612024-12-04T13:32:46.036Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.267235197Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"client":"2","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1622024-12-04T13:32:46.036Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.267251295Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"":"io task","client":"1","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1632024-12-04T13:32:46.036Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.267268763Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"":"io task","client":"0","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1642024-12-04T13:32:46.036Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.267292571Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"":"io task","client":"2","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1652024-12-04T13:32:46.036Z{"msg":"ds_connection connected from Ok(127.0.0.1:48867)","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.267328927Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"":"io task","client":"2","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1662024-12-04T13:32:46.036Z{"msg":"ds_connection connected from Ok(127.0.0.1:40059)","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.267367233Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"":"io task","client":"0","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1672024-12-04T13:32:46.036Z{"msg":"ds_connection connected from Ok(127.0.0.1:35523)","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.267484011Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"":"io task","client":"1","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1682024-12-04T13:32:46.036Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.267996947Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"client":"2","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1692024-12-04T13:32:46.036Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.268040213Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"client":"2","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1702024-12-04T13:32:46.036Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.26815858Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"client":"1","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1712024-12-04T13:32:46.036Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.268199836Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"client":"1","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1722024-12-04T13:32:46.036Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.26826235Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"client":"0","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1732024-12-04T13:32:46.036Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.268300846Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"client":"0","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1742024-12-04T13:32:46.036Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.268654699Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"client":"2","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1752024-12-04T13:32:46.036Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.268899983Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"client":"0","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1762024-12-04T13:32:46.036Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.268998683Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"client":"1","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1772024-12-04T13:32:46.036Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.26912293Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"client":"2","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1782024-12-04T13:32:46.036Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.269157576Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"client":"2","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1792024-12-04T13:32:46.036Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.269173255Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1802024-12-04T13:32:46.037Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.269279704Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"client":"0","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1812024-12-04T13:32:46.037Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.269308901Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"client":"0","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1822024-12-04T13:32:46.037Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.269323579Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1832024-12-04T13:32:46.037Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.269354526Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"client":"1","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1842024-12-04T13:32:46.037Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.269388422Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"client":"1","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1852024-12-04T13:32:46.037Z{"msg":"[0]R flush_numbers: [1496, 1496, 1496, 1496, 1496]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.269422889Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"":"downstairs","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1862024-12-04T13:32:46.037Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.269456995Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"":"downstairs","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1872024-12-04T13:32:46.037Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.269491062Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"":"downstairs","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1882024-12-04T13:32:46.037Z{"msg":"[1]R flush_numbers: [1496, 1496, 1496, 1496, 1496]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.26951079Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226Wait for a query_work_queue command to finish before sending IO
1892024-12-04T13:32:46.037Z,"":"downstairs","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1902024-12-04T13:32:46.037Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.269547976Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"":"downstairs","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1912024-12-04T13:32:46.037Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.269565924Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"":"downstairs","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1922024-12-04T13:32:46.037Z{"msg":"[2]R flush_numbers: [1496, 1496, 1496, 1496, 1496]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.269599001Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"":"downstairs","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1932024-12-04T13:32:46.040Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.269615609Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"":"downstairs","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1942024-12-04T13:32:46.040Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.269648325Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"":"downstairs","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1952024-12-04T13:32:46.040Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.269666383Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"":"downstairs","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1962024-12-04T13:32:46.040Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.26969878Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"":"downstairs","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1972024-12-04T13:32:46.040Z{"msg":"Next flush: 1497","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.269714888Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"":"downstairs","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1982024-12-04T13:32:46.040Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.269728617Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"":"downstairs","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
1992024-12-04T13:32:46.040Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.269742006Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
2002024-12-04T13:32:46.040Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.269754164Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"client":"0","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
2012024-12-04T13:32:46.040Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.269767673Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"client":"1","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
2022024-12-04T13:32:46.040Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.269781142Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"client":"2","":"downstairs","session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
2032024-12-04T13:32:46.040Z{"msg":"All required reconciliation work is completed","v":0,"Disk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2042024-12-04T13:32:46.040Zname":"crucible","level":30,"time":"2024-12-04T13:32:45.26979821Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
2052024-12-04T13:32:46.040Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.269811178Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
2062024-12-04T13:32:46.040Z{"msg":"f486fb3c-423f-4cd3-ac15-e0c5f051c80e is now active with session: e5f0862b-402f-4f0e-8b96-27a77224129d","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.269824297Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
2072024-12-04T13:32:46.040Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Fill test
2082024-12-04T13:32:46.040Z,"time":"2024-12-04T13:32:45.269836546Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226,"session_id":"e5f0862b-402f-4f0e-8b96-27a77224129d"}
2092024-12-04T13:32:46.040Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:45.269859123Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226}
2102024-12-04T13:32:46.043ZRead and Verify all blocks (0..50 range:false)
2112024-12-04T13:32:46.047ZWrote out file "/var/tmp/test_up-build/verify_file"
2122024-12-04T13:32:46.047ZCLIENT: Tests done. All submitted work has been ACK'd
2132024-12-04T13:32:46.049Z----------------------------------------------------------------
2142024-12-04T13:32:46.049Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2152024-12-04T13:32:46.049ZStates: Active Active Active
2162024-12-04T13:32:46.049ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2172024-12-04T13:32:46.049Z 1002 Acked Read 50 Done Done Done false
2182024-12-04T13:32:46.049Z STATES DS:0 DS:1 DS:2 TOTAL
2192024-12-04T13:32:46.049Z Sent 0 0 0 0
2202024-12-04T13:32:46.049Z Done 1 1 1 3
2212024-12-04T13:32:46.049Z Skipped 0 0 0 0
2222024-12-04T13:32:46.049Z Error 0 0 0 0
2232024-12-04T13:32:46.049ZLast Flush: 1001 1001 1001
2242024-12-04T13:32:46.049ZDownstairs last five completed: 1001 1000
2252024-12-04T13:32:46.049ZUpstairs last five completed: 1002 1001 1000
2262024-12-04T13:32:46.049ZCLIENT: Up:0 ds:1 act:3
2272024-12-04T13:32:50.049Z----------------------------------------------------------------
2282024-12-04T13:32:50.049Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2292024-12-04T13:32:50.049ZDownstairs last five completed: 1003 1002 1001 1000
2302024-12-04T13:32:50.049ZUpstairs last five completed: 1003 1002 1001 1000
2312024-12-04T13:32:50.049ZCLIENT: Up:0 ds:0 act:3
2322024-12-04T13:32:50.049ZCLIENT: All crucible jobs finished, exiting program
2332024-12-04T13:32:50.052Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-04T13:32:49.282925381Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1226}
2342024-12-04T13:32:50.061ZRepair setup passed
2352024-12-04T13:32:50.064ZCopy the region for /var/tmp/test_up-build/8830
2362024-12-04T13:32:50.067Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2372024-12-04T13:32:50.070Z/var/tmp/bins/crutest fill -g 32 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2382024-12-04T13:32:50.126Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.359383643Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"task":"crutest"}
2392024-12-04T13:32:50.128Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.359925086Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"task":"crutest"}
2402024-12-04T13:32:50.128Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.361665055Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"task":"crutest"}
2412024-12-04T13:32:50.128Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.361851315Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232}
2422024-12-04T13:32:50.129Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0e1f7f60aff9f26b0d737c97120445edbded1ad3\",\n git_commit_timestamp: \"2024-12-04T13:14:36.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.361889911Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232}
2432024-12-04T13:32:50.129Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.361912279Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232}
2442024-12-04T13:32:50.129Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.361928158Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232}
2452024-12-04T13:32:50.131Z{"msg":"Crucible b4580dea-1bf7-4b08-989d-18cc9213761b has session id: 111758bd-d9d0-4a89-99db-ed5c102c0f85","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.362399398Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2462024-12-04T13:32:50.132Z{"msg":"Upstairs opts: Upstairs UUID: b4580dea-1bf7-4b08-989d-18cc9213761b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.362446623Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2472024-12-04T13:32:50.132Z{"msg":"Crucible stats registered with UUID: b4580dea-1bf7-4b08-989d-18cc9213761b","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.362469901Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2482024-12-04T13:32:50.162Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.395653363Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"task":"crutest"}
2492024-12-04T13:32:50.162Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.395705067Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232}
2502024-12-04T13:32:50.165Z{"msg":"b4580dea-1bf7-4b08-989d-18cc9213761b active request set","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.395730584Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2512024-12-04T13:32:50.165Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.39577061Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"client":"0","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2522024-12-04T13:32:50.165Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.395823865Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"client":"0","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2532024-12-04T13:32:50.165Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.395875299Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"client":"1","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2542024-12-04T13:32:50.165Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.395917555Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"client":"1","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2552024-12-04T13:32:50.165Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.39596342Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"client":"2","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2562024-12-04T13:32:50.165Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.396006586Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"client":"2","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2572024-12-04T13:32:50.165Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.396074499Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2582024-12-04T13:32:50.165Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.396107955Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2592024-12-04T13:32:50.165Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.396226533Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2602024-12-04T13:32:50.165Z{"msg":"ds_connection connected from Ok(127.0.0.1:42084)","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.396268038Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2612024-12-04T13:32:50.165Z{"msg":"ds_connection connected from Ok(127.0.0.1:63454)","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.396294686Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2622024-12-04T13:32:50.165Z{"msg":"ds_connection connected from Ok(127.0.0.1:32777)","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.396322503Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2632024-12-04T13:32:50.165Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.396893643Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"client":"2","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2642024-12-04T13:32:50.165Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.396937858Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"client":"2","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2652024-12-04T13:32:50.165Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.396962816Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"client":"0","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2662024-12-04T13:32:50.165Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.396988923Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"client":"0","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2672024-12-04T13:32:50.166Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.397033639Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"client":"1","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2682024-12-04T13:32:50.166Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.397077744Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"client":"1","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2692024-12-04T13:32:50.166Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.397558254Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"client":"2","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2702024-12-04T13:32:50.166Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.397745394Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"client":"1","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2712024-12-04T13:32:50.166Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.397871801Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"client":"0","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2722024-12-04T13:32:50.166Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.39798109Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"client":"2","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2732024-12-04T13:32:50.166Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.398034134Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"client":"2","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2742024-12-04T13:32:50.166Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.398058472Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2752024-12-04T13:32:50.166Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.398146832Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"client":"1","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2762024-12-04T13:32:50.166Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.398177129Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"client":"1","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2772024-12-04T13:32:50.166Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.398191928Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2782024-12-04T13:32:50.166Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.398229694Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"client":"0","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2792024-12-04T13:32:50.166Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.39826432Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"client":"0","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2802024-12-04T13:32:50.166Z{"msg":"[0]R flush_numbers: [1497, 1497, 1497, 1497, 1497]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.398296007Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"":"downstairs","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2812024-12-04T13:32:50.166Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.398331653Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"":"downstairs","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2822024-12-04T13:32:50.166Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.39836521Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"":"downstairs","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2832024-12-04T13:32:50.166Z{"msg":"[1]R flush_numbers: [1497, 1497, 1497, 1497, 1497]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.398386587Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"":"downstairs","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2842024-12-04T13:32:50.166ZWait for a query_work_queue command to finish before sending IO
2852024-12-04T13:32:50.166Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.398473129Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"":"downstairs","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2862024-12-04T13:32:50.166Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.398508075Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"":"downstairs","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2872024-12-04T13:32:50.166Z{"msg":"[2]R flush_numbers: [1497, 1497, 1497, 1497, 1497]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.398524293Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"":"downstairs","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2882024-12-04T13:32:50.169Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.398538752Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"":"downstairs","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2892024-12-04T13:32:50.169Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.398573108Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"":"downstairs","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2902024-12-04T13:32:50.169Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.398590006Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"":"downstairs","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2912024-12-04T13:32:50.169Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.398622593Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"":"downstairs","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2922024-12-04T13:32:50.169Z{"msg":"Next flush: 1498","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.398639291Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"":"downstairs","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2932024-12-04T13:32:50.169Z{"msg":"All extents match","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2942024-12-04T13:32:50.169Z,"time":"2024-12-04T13:32:49.398672708Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"":"downstairs","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2952024-12-04T13:32:50.169Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.398692476Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2962024-12-04T13:32:50.169Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.398723912Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"client":"0","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2972024-12-04T13:32:50.169Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.398754769Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"client":"1","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2982024-12-04T13:32:50.170Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.398788786Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"client":"2","":"downstairs","session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
2992024-12-04T13:32:50.170Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.398821482Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
3002024-12-04T13:32:50.170Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Loading write count information from file "/var/tmp/test_up-build/verify_file"
3012024-12-04T13:32:50.170Z,"time":"2024-12-04T13:32:49.398837531Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
3022024-12-04T13:32:50.170Z{"msg":"b4580dea-1bf7-4b08-989d-18cc9213761b is now active with session: 111758bd-d9d0-4a89-99db-ed5c102c0f85","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.398865438Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
3032024-12-04T13:32:50.170ZFill test
3042024-12-04T13:32:50.170Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.398894555Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232,"session_id":"111758bd-d9d0-4a89-99db-ed5c102c0f85"}
3052024-12-04T13:32:50.170Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:49.398912433Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232}
3062024-12-04T13:32:50.172ZRead and Verify all blocks (0..50 range:false)
3072024-12-04T13:32:50.176ZWrote out file "/var/tmp/test_up-build/verify_file"
3082024-12-04T13:32:50.176ZCLIENT: Tests done. All submitted work has been ACK'd
3092024-12-04T13:32:50.179Z----------------------------------------------------------------
3102024-12-04T13:32:50.179Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
3112024-12-04T13:32:50.179ZStates: Active Active Active
3122024-12-04T13:32:50.179ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
3132024-12-04T13:32:50.179Z 1002 Acked Read 50 Done Done Done false
3142024-12-04T13:32:50.179Z STATES DS:0 DS:1 DS:2 TOTAL
3152024-12-04T13:32:50.179Z Sent 0 0 0 0
3162024-12-04T13:32:50.179Z Done 1 1 1 3
3172024-12-04T13:32:50.179Z Skipped 0 0 0 0
3182024-12-04T13:32:50.179Z Error 0 0 0 0
3192024-12-04T13:32:50.179ZLast Flush: 1001 1001 1001
3202024-12-04T13:32:50.179ZDownstairs last five completed: 1001 1000
3212024-12-04T13:32:50.179ZUpstairs last five completed: 1002 1001 1000
3222024-12-04T13:32:50.179ZCLIENT: Up:0 ds:1 act:3
3232024-12-04T13:32:54.178Z----------------------------------------------------------------
3242024-12-04T13:32:54.178Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
3252024-12-04T13:32:54.178ZDownstairs last five completed: 1003 1002 1001 1000
3262024-12-04T13:32:54.178ZUpstairs last five completed: 1003 1002 1001 1000
3272024-12-04T13:32:54.178ZCLIENT: Up:0 ds:0 act:3
3282024-12-04T13:32:54.178ZCLIENT: All crucible jobs finished, exiting program
3292024-12-04T13:32:54.180Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-04T13:32:53.411255536Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1232}
3302024-12-04T13:32:54.185ZRepair part 1 passed
3312024-12-04T13:32:54.187Z
3322024-12-04T13:32:54.187ZKill the current downstairs
3332024-12-04T13:32:54.351ZDownstairs 2 stopped
3342024-12-04T13:32:54.352Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
3352024-12-04T13:32:54.354ZNow put back the original so we have a mismatch
3362024-12-04T13:32:54.354Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
3372024-12-04T13:32:54.357ZRestart downstairs with old directory
3382024-12-04T13:32:54.409Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
3392024-12-04T13:32:54.424ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3402024-12-04T13:32:54.427Z 0 000-009 32 32 31 1498 1498 1497 F F F <---
3412024-12-04T13:32:54.427Z 1 010-019 32 32 31 1498 1498 1497 F F F <---
3422024-12-04T13:32:54.427Z 2 020-029 32 32 31 1498 1498 1497 F F F <---
3432024-12-04T13:32:54.427Z 3 030-039 32 32 31 1498 1498 1497 F F F <---
3442024-12-04T13:32:54.427Z 4 040-049 32 32 31 1498 1498 1497 F F F <---
3452024-12-04T13:32:54.427ZMax gen: 32, Max flush: 1498
3462024-12-04T13:32:54.427ZError: Difference in extent metadata found!
3472024-12-04T13:32:54.429Zdump test found error as expected
3482024-12-04T13:32:54.429Z
3492024-12-04T13:32:54.429Z
3502024-12-04T13:32:54.429Z/var/tmp/bins/crutest verify --range -g 33 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
3512024-12-04T13:32:54.487Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.719832915Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"task":"crutest"}
3522024-12-04T13:32:54.490Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.720365469Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"task":"crutest"}
3532024-12-04T13:32:54.490Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.722046694Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"task":"crutest"}
3542024-12-04T13:32:54.490Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.722240424Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240}
3552024-12-04T13:32:54.490Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0e1f7f60aff9f26b0d737c97120445edbded1ad3\",\n git_commit_timestamp: \"2024-12-04T13:14:36.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.72227952Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240}
3562024-12-04T13:32:54.490Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.722295808Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240}
3572024-12-04T13:32:54.490Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.722319246Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240}
3582024-12-04T13:32:54.490Z{"msg":"Crucible 4cf7aac0-7898-459d-9c00-85b071f48e53 has session id: 48026ff9-331a-47c3-bb19-98dd71fe031a","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.722733253Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3592024-12-04T13:32:54.490Z{"msg":"Upstairs opts: Upstairs UUID: 4cf7aac0-7898-459d-9c00-85b071f48e53, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.722785277Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3602024-12-04T13:32:54.490Z{"msg":"Crucible stats registered with UUID: 4cf7aac0-7898-459d-9c00-85b071f48e53","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.722811944Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3612024-12-04T13:32:54.523Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.756889573Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"task":"crutest"}
3622024-12-04T13:32:54.523Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.756935028Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240}
3632024-12-04T13:32:54.526Z{"msg":"4cf7aac0-7898-459d-9c00-85b071f48e53 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.756957955Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3642024-12-04T13:32:54.526Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.756992372Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"client":"0","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3652024-12-04T13:32:54.526Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.757028908Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"client":"0","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3662024-12-04T13:32:54.527Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.757066894Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"client":"1","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3672024-12-04T13:32:54.527Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.757111379Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"client":"1","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3682024-12-04T13:32:54.527Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.757153835Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"client":"2","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3692024-12-04T13:32:54.527Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.757195811Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"client":"2","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3702024-12-04T13:32:54.527Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.757214209Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3712024-12-04T13:32:54.527Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.757255264Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3722024-12-04T13:32:54.527Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.757282601Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3732024-12-04T13:32:54.527Z{"msg":"ds_connection connected from Ok(127.0.0.1:35641)","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.757320008Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3742024-12-04T13:32:54.527Z{"msg":"ds_connection connected from Ok(127.0.0.1:51062)","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.757348185Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3752024-12-04T13:32:54.527Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-12-04T13:32:53.757376492Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3762024-12-04T13:32:54.527Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-12-04T13:32:53.757414218Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3772024-12-04T13:32:54.527Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-12-04T13:32:53.75758851Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3782024-12-04T13:32:54.527Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.757701198Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3792024-12-04T13:32:54.527Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.758074379Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"client":"1","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3802024-12-04T13:32:54.527Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.75815752Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"client":"1","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3812024-12-04T13:32:54.527Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.758226773Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"client":"0","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3822024-12-04T13:32:54.527Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.758280717Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"client":"0","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3832024-12-04T13:32:54.527Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.75873525Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"client":"1","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3842024-12-04T13:32:54.527Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.758860197Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"client":"0","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3852024-12-04T13:32:54.527Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.759087164Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"client":"1","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3862024-12-04T13:32:54.527Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.759144658Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"client":"1","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3872024-12-04T13:32:54.527Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.759178954Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3882024-12-04T13:32:54.527Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.759193693Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"client":"0","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3892024-12-04T13:32:54.527Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.75921553Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"client":"0","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3902024-12-04T13:32:54.527Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:32:53.759230019Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3912024-12-04T13:33:04.527Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.759419419Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3922024-12-04T13:33:04.532Z{"msg":"ds_connection connected from Ok(127.0.0.1:39539)","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.75959896Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3932024-12-04T13:33:04.532Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.760428034Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"client":"2","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3942024-12-04T13:33:04.532Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.760469319Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"client":"2","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3952024-12-04T13:33:04.532Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.760969607Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"client":"2","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3962024-12-04T13:33:04.532Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.76132887Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"client":"2","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3972024-12-04T13:33:04.532Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.761363276Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"client":"2","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3982024-12-04T13:33:04.532Z{"msg":"[0]R flush_numbers: [1498, 1498, 1498, 1498, 1498]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.761380904Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
3992024-12-04T13:33:04.532Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.761403922Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4002024-12-04T13:33:04.532Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.76141892Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4012024-12-04T13:33:04.532Z{"msg":"[1]R flush_numbers: [1498, 1498, 1498, 1498, 1498]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.761441438Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4022024-12-04T13:33:04.532Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.761456396Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4032024-12-04T13:33:04.532Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.761478434Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4042024-12-04T13:33:04.532Z{"msg":"[2]R flush_numbers: [1497, 1497, 1497, 1497, 1497]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.761493313Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4052024-12-04T13:33:04.532Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.761507221Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4062024-12-04T13:33:04.532Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.761528919Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4072024-12-04T13:33:04.532Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.761543427Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4082024-12-04T13:33:04.532Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.761564455Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4092024-12-04T13:33:04.532Z{"msg":"Next flush: 1499","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.761578774Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4102024-12-04T13:33:04.532Z{"msg":"Extent 0 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.761600181Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4112024-12-04T13:33:04.532Z{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.7616151Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4122024-12-04T13:33:04.532Z{"msg":"extent:0 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.761637538Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4132024-12-04T13:33:04.532Z{"msg":"extent:0 flush: 1498 1498 1497 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.761653816Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4142024-12-04T13:33:04.532Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.761668904Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4152024-12-04T13:33:04.532Z{"msg":"extent:0 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.761691172Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4162024-12-04T13:33:04.532Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.76170685Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4172024-12-04T13:33:04.533Z{"msg":"find dest for source 0 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.761728838Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4182024-12-04T13:33:04.533Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.761744086Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4192024-12-04T13:33:04.533Z{"msg":"Extent 1 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.761777553Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4202024-12-04T13:33:04.533Z{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.76180162Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4212024-12-04T13:33:04.533Z{"msg":"extent:1 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.761818229Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4222024-12-04T13:33:04.533Z{"msg":"extent:1 flush: 1498 1498 1497 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.761841256Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4232024-12-04T13:33:04.533Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.761856975Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4242024-12-04T13:33:04.533Z{"msg":"extent:1 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.761872863Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4252024-12-04T13:33:04.533Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.761888031Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4262024-12-04T13:33:04.533Z{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.761915109Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4272024-12-04T13:33:04.533Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.761955084Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4282024-12-04T13:33:04.533Z{"msg":"Extent 2 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.762002859Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4292024-12-04T13:33:04.533Z{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.762044815Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4302024-12-04T13:33:04.533Z{"msg":"extent:2 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.762081002Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4312024-12-04T13:33:04.533Z{"msg":"extent:2 flush: 1498 1498 1497 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.762116968Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4322024-12-04T13:33:04.533Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.762159253Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4332024-12-04T13:33:04.533Z{"msg":"extent:2 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.76219633Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4342024-12-04T13:33:04.533Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.762234965Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4352024-12-04T13:33:04.533Z{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.762277481Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4362024-12-04T13:33:04.533Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.762315817Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4372024-12-04T13:33:04.533Z{"msg":"Extent 3 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.762334735Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4382024-12-04T13:33:04.533Z{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.762356843Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4392024-12-04T13:33:04.533Z{"msg":"extent:3 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.762372381Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4402024-12-04T13:33:04.533Z{"msg":"extent:3 flush: 1498 1498 1497 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.762387939Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4412024-12-04T13:33:04.533Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.762404328Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4422024-12-04T13:33:04.536Z{"msg":"extent:3 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.762443644Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4432024-12-04T13:33:04.536Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.76247533Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4442024-12-04T13:33:04.536Z{"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.762526495Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4452024-12-04T13:33:04.536Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.762563161Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4462024-12-04T13:33:04.536Z{"msg":"Extent 4 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.762596998Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4472024-12-04T13:33:04.536Z{"msg":"First source client ID for extent 4","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.762624965Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4482024-12-04T13:33:04.536Z{"msg":"extent:4 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.762648352Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4492024-12-04T13:33:04.536Z{"msg":"extent:4 flush: 1498 1498 1497 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.762688678Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4502024-12-04T13:33:04.536Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.762708876Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4512024-12-04T13:33:04.536Z{"msg":"extent:4 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.762732774Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4522024-12-04T13:33:04.536Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.762748692Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4532024-12-04T13:33:04.536Z{"msg":"find dest for source 0 for extent at index 4","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.76276382Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4542024-12-04T13:33:04.536Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.762786308Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4552024-12-04T13:33:04.536Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.762808166Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"client":"0","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4562024-12-04T13:33:04.536Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.762842282Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"client":"1","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4572024-12-04T13:33:04.536Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.76285879Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"client":"2","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4582024-12-04T13:33:04.536Z{"msg":"Full repair list: {ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.762877878Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4592024-12-04T13:33:04.536Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 1499, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 1499, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 1499, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 1499, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 1499, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.76295735Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4602024-12-04T13:33:04.536Z{"msg":"starting reconciliation 52d3872a-863d-4fb4-9eaa-e1f30ccd0910: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.762974448Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4612024-12-04T13:33:04.537Z{"msg":"no Nexus client from DNS, aborting start notification","v":0,"name":"crucible","level":50,"time":"2024-12-04T13:33:03.762988557Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"reconcile":"52d3872a-863d-4fb4-9eaa-e1f30ccd0910","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4622024-12-04T13:33:04.537Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-04T13:33:03.763012294Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"reconcile":"52d3872a-863d-4fb4-9eaa-e1f30ccd0910","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4632024-12-04T13:33:04.537Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-04T13:33:03.763028463Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"reconcile":"52d3872a-863d-4fb4-9eaa-e1f30ccd0910","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4642024-12-04T13:33:04.537Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.76305121Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"client":"2","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4652024-12-04T13:33:04.537Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-04T13:33:03.763065569Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"reconcile":"52d3872a-863d-4fb4-9eaa-e1f30ccd0910","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4662024-12-04T13:33:04.539Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-04T13:33:03.770068149Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"reconcile":"52d3872a-863d-4fb4-9eaa-e1f30ccd0910","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4672024-12-04T13:33:04.539Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-04T13:33:03.770518352Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"reconcile":"52d3872a-863d-4fb4-9eaa-e1f30ccd0910","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4682024-12-04T13:33:04.539Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-04T13:33:03.770774036Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"reconcile":"52d3872a-863d-4fb4-9eaa-e1f30ccd0910","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4692024-12-04T13:33:04.539Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.771084403Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"client":"2","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4702024-12-04T13:33:04.539Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-04T13:33:03.771126019Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"reconcile":"52d3872a-863d-4fb4-9eaa-e1f30ccd0910","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4712024-12-04T13:33:04.542Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-04T13:33:03.773819587Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"reconcile":"52d3872a-863d-4fb4-9eaa-e1f30ccd0910","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4722024-12-04T13:33:04.545Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-04T13:33:03.774243783Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"reconcile":"52d3872a-863d-4fb4-9eaa-e1f30ccd0910","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4732024-12-04T13:33:04.545Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-04T13:33:03.774499156Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"reconcile":"52d3872a-863d-4fb4-9eaa-e1f30ccd0910","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4742024-12-04T13:33:04.545Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.774825532Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"client":"2","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4752024-12-04T13:33:04.545Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-04T13:33:03.774860039Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"reconcile":"52d3872a-863d-4fb4-9eaa-e1f30ccd0910","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4762024-12-04T13:33:04.548Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-04T13:33:03.777574726Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"reconcile":"52d3872a-863d-4fb4-9eaa-e1f30ccd0910","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4772024-12-04T13:33:04.548Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-04T13:33:03.77801102Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"reconcile":"52d3872a-863d-4fb4-9eaa-e1f30ccd0910","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4782024-12-04T13:33:04.548Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-04T13:33:03.778261764Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"reconcile":"52d3872a-863d-4fb4-9eaa-e1f30ccd0910","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4792024-12-04T13:33:04.548Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.778603439Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"client":"2","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4802024-12-04T13:33:04.548Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-04T13:33:03.778643404Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"reconcile":"52d3872a-863d-4fb4-9eaa-e1f30ccd0910","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4812024-12-04T13:33:04.550Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-04T13:33:03.781438973Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"reconcile":"52d3872a-863d-4fb4-9eaa-e1f30ccd0910","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4822024-12-04T13:33:04.551Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-04T13:33:03.781791796Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"reconcile":"52d3872a-863d-4fb4-9eaa-e1f30ccd0910","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4832024-12-04T13:33:04.551Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-04T13:33:03.782052739Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"reconcile":"52d3872a-863d-4fb4-9eaa-e1f30ccd0910","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4842024-12-04T13:33:04.551Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.782391884Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"client":"2","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4852024-12-04T13:33:04.551Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-04T13:33:03.782443218Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"reconcile":"52d3872a-863d-4fb4-9eaa-e1f30ccd0910","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4862024-12-04T13:33:04.554Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-04T13:33:03.785280163Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"reconcile":"52d3872a-863d-4fb4-9eaa-e1f30ccd0910","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4872024-12-04T13:33:04.554Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.785641715Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4882024-12-04T13:33:04.554Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.785680201Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"client":"0","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4892024-12-04T13:33:04.554Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.785699969Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"client":"1","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4902024-12-04T13:33:04.554Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.785735565Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"client":"2","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4912024-12-04T13:33:04.554Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.785762582Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4922024-12-04T13:33:04.554Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.785800978Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4932024-12-04T13:33:04.554Z{"msg":"4cf7aac0-7898-459d-9c00-85b071f48e53 is now active with session: 48026ff9-331a-47c3-bb19-98dd71fe031a","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.785830455Z","Wait for a query_work_queue command to finish before sending IO
4942024-12-04T13:33:04.554Zhostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4952024-12-04T13:33:04.554Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2024-12-04T13:33:03.78587523Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240}
4962024-12-04T13:33:04.554Z{"msg":"no Nexus client from DNS, aborting finish notification","v":0,"name":"crucible","level":50,"time":"2024-12-04T13:33:03.785912557Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240,"reconcile":"52d3872a-863d-4fb4-9eaa-e1f30ccd0910","":"downstairs","":"downstairs","session_id":"48026ff9-331a-47c3-bb19-98dd71fe031a"}
4972024-12-04T13:33:04.554ZDisk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
4982024-12-04T13:33:04.555ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4992024-12-04T13:33:04.555ZRead and Verify all blocks (0..50 range:true)
5002024-12-04T13:33:04.561ZWrote out file "/var/tmp/test_up-build/verify_file"
5012024-12-04T13:33:04.561ZVerify test completed
5022024-12-04T13:33:04.564ZWrote out file "/var/tmp/test_up-build/verify_file"
5032024-12-04T13:33:04.564ZCLIENT: Tests done. All submitted work has been ACK'd
5042024-12-04T13:33:04.564Z----------------------------------------------------------------
5052024-12-04T13:33:04.564Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
5062024-12-04T13:33:04.564ZStates: Active Active Active
5072024-12-04T13:33:04.564ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
5082024-12-04T13:33:04.564Z 1000 Acked Read 50 Done Done Done false
5092024-12-04T13:33:04.564Z STATES DS:0 DS:1 DS:2 TOTAL
5102024-12-04T13:33:04.564Z Sent 0 0 0 0
5112024-12-04T13:33:04.564Z Done 1 1 1 3
5122024-12-04T13:33:04.564Z Skipped 0 0 0 0
5132024-12-04T13:33:04.564Z Error 0 0 0 0
5142024-12-04T13:33:04.564ZLast Flush: 0 0 0
5152024-12-04T13:33:04.564ZDownstairs last five completed:
5162024-12-04T13:33:04.564ZUpstairs last five completed: 1000
5172024-12-04T13:33:04.564ZCLIENT: Up:0 ds:1 act:3
5182024-12-04T13:33:08.564Z----------------------------------------------------------------
5192024-12-04T13:33:08.564Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
5202024-12-04T13:33:08.564ZDownstairs last five completed: 1001 1000
5212024-12-04T13:33:08.564ZUpstairs last five completed: 1001 1000
5222024-12-04T13:33:08.564ZCLIENT: Up:0 ds:0 act:3
5232024-12-04T13:33:08.564ZCLIENT: All crucible jobs finished, exiting program
5242024-12-04T13:33:08.566Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-04T13:33:07.795313246Z","hostname":"w-01JE8XRS07NFFENGZRQFRBMQZH","pid":1240}
5252024-12-04T13:33:08.571ZRepair part 2 passed
5262024-12-04T13:33:08.574Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
5272024-12-04T13:33:08.588ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
5282024-12-04T13:33:08.588Z 0 000-009 32 32 32 1498 1498 1498 F F F
5292024-12-04T13:33:08.588Z 1 010-019 32 32 32 1498 1498 1498 F F F
5302024-12-04T13:33:08.588Z 2 020-029 32 32 32 1498 1498 1498 F F F
5312024-12-04T13:33:08.588Z 3 030-039 32 32 32 1498 1498 1498 F F F
5322024-12-04T13:33:08.588Z 4 040-049 32 32 32 1498 1498 1498 F F F
5332024-12-04T13:33:08.588ZMax gen: 32, Max flush: 1498
5342024-12-04T13:33:08.591Zdump test passed
5352024-12-04T13:33:08.594Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
5362024-12-04T13:33:08.606Z Extent 1
5372024-12-04T13:33:08.606ZGEN 32 32 32
5382024-12-04T13:33:08.606ZFLUSH_ID 1498 1498 1498
5392024-12-04T13:33:08.606ZDIRTY
5402024-12-04T13:33:08.606Z
5412024-12-04T13:33:08.606ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
5422024-12-04T13:33:08.608Z010 A A A A A A
5432024-12-04T13:33:08.611Z011 A A A A A A
5442024-12-04T13:33:08.611Z012 A A A A A A
5452024-12-04T13:33:08.613Z013 A A A A A A
5462024-12-04T13:33:08.613Z014 A A A A A A
5472024-12-04T13:33:08.613Z015 A A A A A A
5482024-12-04T13:33:08.616Z016 A A A A A A
5492024-12-04T13:33:08.616Z017 A A A A A A
5502024-12-04T13:33:08.619Z018 A A A A A A
5512024-12-04T13:33:08.619Z019 A A A A A A
5522024-12-04T13:33:08.621Zdump extent test passed
5532024-12-04T13:33:08.624Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
5542024-12-04T13:33:08.635ZExtent 2 Block in extent 0 Actual block 20
5552024-12-04T13:33:08.635Z
5562024-12-04T13:33:08.638Z DATA SHA256 VER
5572024-12-04T13:33:08.638Z------ ---------------------------------------------------------------- ---
5582024-12-04T13:33:08.638Z 0 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
5592024-12-04T13:33:08.638Z 1 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
5602024-12-04T13:33:08.638Z 2 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
5612024-12-04T13:33:08.638Z
5622024-12-04T13:33:08.638ZNONCES 0 1 2 DIFF
5632024-12-04T13:33:08.638Z------ ------------------------ ------------------------ ------------------------ -----
5642024-12-04T13:33:08.638Z
5652024-12-04T13:33:08.638Z TAGS 0 1 2 DIFF
5662024-12-04T13:33:08.638Z------ -------------------------------- -------------------------------- -------------------------------- -----
5672024-12-04T13:33:08.638Z
5682024-12-04T13:33:08.638ZHASHES 0 1 2 DIFF
5692024-12-04T13:33:08.638Z------ ---------------- ---------------- ---------------- -----
5702024-12-04T13:33:08.641Z 0 42916ab6fa364c70 42916ab6fa364c70 42916ab6fa364c70
5712024-12-04T13:33:08.641Z
5722024-12-04T13:33:08.641Zdump block test passed
5732024-12-04T13:33:08.641ZInitial upstairs tests have completed, stopping all downstairs
5742024-12-04T13:33:13.705ZCreating 4 larger downstairs regions
5752024-12-04T13:33:13.815ZStarting 4 downstairs
5762024-12-04T13:33:18.822Zdsc restarted at PID: 1253
5772024-12-04T13:33:19.045ZNow do the replace-reconcile test
5782024-12-04T13:33:19.100ZUsing 8840 for the replacement port
5792024-12-04T13:37:12.758ZCompleted test: replace-reconcile
5802024-12-04T13:37:12.760ZNow do the replace-before-active test
5812024-12-04T13:39:57.440ZCompleted test: replace-before-active
5822024-12-04T13:39:57.440ZAll tests have completed, stopping all downstairs
5832024-12-04T13:39:57.498Z
5842024-12-04T13:39:57.504ZAll Tests have passed
5852024-12-04T13:39:57.504Z9:10 Test duration
5862024-12-04T13:39:57.504Z
5872024-12-04T13:39:57.504Zreal 9:10.090711788
5882024-12-04T13:39:57.504Zuser 1:04.686455488
5892024-12-04T13:39:57.504Zsys 9.999944926
5902024-12-04T13:39:57.504Ztrap 0.091264186
5912024-12-04T13:39:57.504Ztflt 0.016879700
5922024-12-04T13:39:57.504Zdflt 0.013794346
5932024-12-04T13:39:57.504Zkflt 0.001162868
5942024-12-04T13:39:57.504Zlock 6:05:22.752508734
5952024-12-04T13:39:57.504Zslp 27:08.410321268
5962024-12-04T13:39:57.504Zlat 8.048153686
5972024-12-04T13:39:57.504Zstop 0.125786259
5982024-12-04T13:39:57.504Z+ echo 'test-up-unencrypted ends'
5992024-12-04T13:39:57.504Ztest-up-unencrypted ends
6002024-12-04T13:40:02.503Zprocess exited: duration 558326 ms, exit code 0
6012024-12-04T13:40:02.503Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
6022024-12-04T13:41:02.535Zexec warning: : stdout descriptor may be held open by a background process; giving up!
6032024-12-04T13:41:02.535Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
6042024-12-04T13:41:02.545Zfound 14 output files
6052024-12-04T13:41:02.545Zuploading: /tmp/test_up-build/dsc-out.txt (46821 bytes)
6062024-12-04T13:41:03.556Zuploaded: /tmp/test_up-build/dsc-out.txt
6072024-12-04T13:41:03.558Zuploading: /tmp/test_up-build/test_up_out.txt (11804884 bytes)
6082024-12-04T13:41:04.718Zuploaded: /tmp/test_up-build/test_up_out.txt
6092024-12-04T13:41:04.720Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (2507 bytes)
6102024-12-04T13:41:05.726Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
6112024-12-04T13:41:05.730Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (144653 bytes)
6122024-12-04T13:41:06.746Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
6132024-12-04T13:41:06.748Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (251114 bytes)
6142024-12-04T13:41:07.776Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
6152024-12-04T13:41:07.778Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (139426 bytes)
6162024-12-04T13:41:08.793Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
6172024-12-04T13:41:08.795Zuploading: /tmp/debug/df.txt (1270 bytes)
6182024-12-04T13:41:09.801Zuploaded: /tmp/debug/df.txt
6192024-12-04T13:41:09.803Zuploading: /tmp/debug/dtrace.txt (582125 bytes)
6202024-12-04T13:41:10.833Zuploaded: /tmp/debug/dtrace.txt
6212024-12-04T13:41:10.833Zuploading: /tmp/debug/iostat.txt (185426 bytes)
6222024-12-04T13:41:10.848Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 185426 -> 187834
6232024-12-04T13:41:11.854Zuploaded: /tmp/debug/iostat.txt
6242024-12-04T13:41:11.857Zuploading: /tmp/debug/mpstat.txt (449687 bytes)
6252024-12-04T13:41:11.880Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 449687 -> 456257
6262024-12-04T13:41:12.886Zuploaded: /tmp/debug/mpstat.txt
6272024-12-04T13:41:12.889Zuploading: /tmp/debug/paging.txt (81252 bytes)
6282024-12-04T13:41:12.897Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 81252 -> 82570
6292024-12-04T13:41:13.903Zuploaded: /tmp/debug/paging.txt
6302024-12-04T13:41:13.905Zuploading: /tmp/debug/prstat.txt (846936 bytes)
6312024-12-04T13:41:13.939Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 846936 -> 861195
6322024-12-04T13:41:14.944Zuploaded: /tmp/debug/prstat.txt
6332024-12-04T13:41:14.947Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
6342024-12-04T13:41:15.952Zuploaded: /tmp/debug/psrinfo.txt
6352024-12-04T13:41:15.952Zuploading: /tmp/debug/upstairs-info.txt (81624 bytes)
6362024-12-04T13:41:16.965Zuploaded: /tmp/debug/upstairs-info.txt