01JBM7T50K0HFKTR9R57X2D8N1: test-up-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01JBM7W24WJT2YKQEE7XAK0XZY

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-11-01T16:24:50.584Zjob dependencies complete; ready to run (waiting for 14 m 23 s)
22024-11-01T16:38:45.046Zjob assigned to worker 01JBM9EHP08T9WT7D3HWEXQS78 (queued for 13 m 54 s)
32024-11-01T16:38:52.678Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-11-01T16:38:53.770Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-11-01T16:38:53.770Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-11-01T16:38:54.828Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-11-01T16:38:54.828Zdownloading input: /input/build/work/bins/crutest.gz
82024-11-01T16:38:55.980Zdownloaded input: /input/build/work/bins/crutest.gz
92024-11-01T16:38:55.980Zdownloading input: /input/build/work/bins/dsc.gz
102024-11-01T16:38:56.851Zdownloaded input: /input/build/work/bins/dsc.gz
112024-11-01T16:38:56.854Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-11-01T16:38:56.871Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-11-01T16:38:56.872Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-11-01T16:38:56.888Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-11-01T16:38:56.888Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-11-01T16:38:56.903Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-11-01T16:38:56.903Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-11-01T16:38:56.923Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-11-01T16:38:56.924Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-11-01T16:38:56.943Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-11-01T16:38:56.943Zdownloading input: /input/build/work/scripts/test_up.sh
222024-11-01T16:38:56.964Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-11-01T16:38:56.965Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-11-01T16:38:56.987Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-11-01T16:38:56.992Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-11-01T16:38:57.727Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-11-01T16:38:57.727Zstarting task 0: "setup"
282024-11-01T16:38:57.735Z++ uname -s
292024-11-01T16:38:57.744Z+ kern=SunOS
302024-11-01T16:38:57.744Z+ case "$kern" in
312024-11-01T16:38:57.744Z+ [[ 0 == 12345 ]]
322024-11-01T16:38:57.744Z+ groupadd -g 12345 build
332024-11-01T16:38:57.744Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
342024-11-01T16:38:59.750Z+ zfs create -o mountpoint=/work rpool/work
352024-11-01T16:38:59.940Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
362024-11-01T16:38:59.942Z+ home_fs=zfs
372024-11-01T16:38:59.943Z+ [[ zfs == autofs ]]
382024-11-01T16:38:59.943Z+ mkdir -p /home/build
392024-11-01T16:38:59.945Z+ chown build:build /home/build /work
402024-11-01T16:39:01.946Z+ chmod 0700 /home/build /work
412024-11-01T16:39:01.949Zprocess exited: duration 4228 ms, exit code 0
 
422024-11-01T16:39:01.955Zstarting task 1: "authentication"
432024-11-01T16:39:01.972Zprocess exited: duration 16 ms, exit code 0
 
442024-11-01T16:39:01.979Zstarting task 2: "build"
452024-11-01T16:39:01.984Z+ banner cores
462024-11-01T16:39:01.987Z
472024-11-01T16:39:01.987Z #### #### ##### ###### ####
482024-11-01T16:39:01.987Z # # # # # # # #
492024-11-01T16:39:01.987Z # # # # # ##### ####
502024-11-01T16:39:01.987Z # # # ##### # #
512024-11-01T16:39:01.988Z # # # # # # # # #
522024-11-01T16:39:01.988Z #### #### # # ###### ####
532024-11-01T16:39:01.988Z
542024-11-01T16:39:01.990Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
552024-11-01T16:39:01.998Z+ echo 'input bins dir contains:'
562024-11-01T16:39:01.998Zinput bins dir contains:
572024-11-01T16:39:02.001Z+ ls -ltr /input/build/work/bins
582024-11-01T16:39:02.001Ztotal 761017
592024-11-01T16:39:02.001Z-rw-r--r-- 1 root root 108991184 Nov 1 16:38 crucible-downstairs.gz
602024-11-01T16:39:02.002Z-rw-r--r-- 1 root root 104639051 Nov 1 16:38 crucible-hammer.gz
612024-11-01T16:39:02.002Z-rw-r--r-- 1 root root 109428757 Nov 1 16:38 crutest.gz
622024-11-01T16:39:02.002Z-rw-r--r-- 1 root root 66334129 Nov 1 16:38 dsc.gz
632024-11-01T16:39:02.002Z+ banner unpack
642024-11-01T16:39:02.005Z
652024-11-01T16:39:02.005Z # # # # ##### ## #### # #
662024-11-01T16:39:02.005Z # # ## # # # # # # # # #
672024-11-01T16:39:02.005Z # # # # # # # # # # ####
682024-11-01T16:39:02.005Z # # # # # ##### ###### # # #
692024-11-01T16:39:02.005Z # # # ## # # # # # # #
702024-11-01T16:39:02.005Z #### # # # # # #### # #
712024-11-01T16:39:02.005Z
722024-11-01T16:39:02.005Z+ mkdir -p /var/tmp/bins
732024-11-01T16:39:02.005Z+ for t in "$input/bins/"*.gz
742024-11-01T16:39:02.008Z++ basename /input/build/work/bins/crucible-downstairs.gz
752024-11-01T16:39:02.008Z+ b=crucible-downstairs.gz
762024-11-01T16:39:02.008Z+ b=crucible-downstairs
772024-11-01T16:39:02.008Z+ gunzip
782024-11-01T16:39:04.246Z+ chmod +x /var/tmp/bins/crucible-downstairs
792024-11-01T16:39:04.248Z+ for t in "$input/bins/"*.gz
802024-11-01T16:39:04.251Z++ basename /input/build/work/bins/crucible-hammer.gz
812024-11-01T16:39:04.251Z+ b=crucible-hammer.gz
822024-11-01T16:39:04.251Z+ b=crucible-hammer
832024-11-01T16:39:04.251Z+ gunzip
842024-11-01T16:39:06.396Z+ chmod +x /var/tmp/bins/crucible-hammer
852024-11-01T16:39:06.399Z+ for t in "$input/bins/"*.gz
862024-11-01T16:39:06.403Z++ basename /input/build/work/bins/crutest.gz
872024-11-01T16:39:06.403Z+ b=crutest.gz
882024-11-01T16:39:06.403Z+ b=crutest
892024-11-01T16:39:06.403Z+ gunzip
902024-11-01T16:39:08.693Z+ chmod +x /var/tmp/bins/crutest
912024-11-01T16:39:08.696Z+ for t in "$input/bins/"*.gz
922024-11-01T16:39:08.699Z++ basename /input/build/work/bins/dsc.gz
932024-11-01T16:39:08.699Z+ b=dsc.gz
942024-11-01T16:39:08.699Z+ b=dsc
952024-11-01T16:39:08.699Z+ gunzip
962024-11-01T16:39:10.102Z+ chmod +x /var/tmp/bins/dsc
972024-11-01T16:39:10.104Z+ export BINDIR=/var/tmp/bins
982024-11-01T16:39:10.104Z+ BINDIR=/var/tmp/bins
992024-11-01T16:39:10.104Z+ jobpid=1137
1002024-11-01T16:39:10.107Z+ echo 'Setup debug logging'
1012024-11-01T16:39:10.107ZSetup debug logging
1022024-11-01T16:39:10.107Z+ mkdir /tmp/debug
1032024-11-01T16:39:10.107Z+ sleep 3600
1042024-11-01T16:39:10.109Z+ psrinfo -v
1052024-11-01T16:39:10.112Z+ df -h
1062024-11-01T16:39:10.118Z+ prstat -d d -mLc 1
1072024-11-01T16:39:10.121Z+ iostat -T d -xn 1
1082024-11-01T16:39:10.121Z+ mpstat -T d 1
1092024-11-01T16:39:10.121Z+ vmstat -T d -p 1
1102024-11-01T16:39:10.123Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1112024-11-01T16:39:10.123Z+ banner test_up_encrypted
1122024-11-01T16:39:10.123Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1132024-11-01T16:39:10.126Z
1142024-11-01T16:39:10.126Z ##### ###### #### ##### # # ##### ###### # #
1152024-11-01T16:39:10.126Z # # # # # # # # # ## #
1162024-11-01T16:39:10.126Z # ##### #### # # # # # ##### # # #
1172024-11-01T16:39:10.126Z # # # # # # ##### # # # #
1182024-11-01T16:39:10.126Z # # # # # # # # # # ##
1192024-11-01T16:39:10.126Z # ###### #### # ####### #### # ####### ###### # #
1202024-11-01T16:39:10.126Z
1212024-11-01T16:39:10.126Z+ ptime -m bash /input/build/work/scripts/test_up.sh -N encrypted
1222024-11-01T16:39:10.136Z/input/build/work
1232024-11-01T16:39:10.149ZTurn off color for downstairs dump
1242024-11-01T16:39:10.172ZUpstairs using key: 91iihcUukm8TDD022wDf1OUglF26JChKuknMSR++tUY=
1252024-11-01T16:39:10.174Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1262024-11-01T16:39:10.174ZCreating 3 downstairs regions
1272024-11-01T16:39:10.487ZStarting 3 downstairs
1282024-11-01T16:39:10.490Zdsc started at PID: 1183
1292024-11-01T16:39:15.661ZDisable automatic restart on all downstairs
1302024-11-01T16:39:15.713Z
1312024-11-01T16:39:15.713ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1322024-11-01T16:39:15.717ZRunning test: span
1332024-11-01T16:39:19.900ZCompleted test: span
1342024-11-01T16:39:24.908ZRunning test: big
1352024-11-01T16:39:29.308ZCompleted test: big
1362024-11-01T16:39:34.326ZRunning test: dep
1372024-11-01T16:40:26.213ZCompleted test: dep
1382024-11-01T16:40:31.219ZRunning test: balloon
1392024-11-01T16:41:15.191ZCompleted test: balloon
1402024-11-01T16:41:20.198ZRunning test: deactivate
1412024-11-01T16:42:14.970ZCompleted test: deactivate
1422024-11-01T16:42:19.980ZRunning hammer
1432024-11-01T16:42:33.288ZRun repair tests
1442024-11-01T16:42:33.291Z/var/tmp/bins/crutest fill -g 31 -q --key 91iihcUukm8TDD022wDf1OUglF26JChKuknMSR++tUY= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1452024-11-01T16:42:33.351Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.327844884Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"task":"crutest"}
1462024-11-01T16:42:33.354Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.328459873Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"task":"crutest"}
1472024-11-01T16:42:33.354Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.330299711Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"task":"crutest"}
1482024-11-01T16:42:33.354Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.330487762Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220}
1492024-11-01T16:42:33.354Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7796cc7e5b7a12d521784f7b6b2dd4b8dd6e6d5b\",\n git_commit_timestamp: \"2024-11-01T16:07:40.000000000Z\",\n git_branch: \"renovate/reqwest-0.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.330551646Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220}
1502024-11-01T16:42:33.354Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.330581653Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220}
1512024-11-01T16:42:33.354Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.330600831Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220}
1522024-11-01T16:42:33.357Z{"msg":"Crucible fc446cdc-ec96-41c3-95bf-a8624bee9e2e has session id: d5214ddb-2451-4c7e-a963-a0e97d5decd6","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.331082364Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1532024-11-01T16:42:33.357Z{"msg":"Upstairs opts: Upstairs UUID: fc446cdc-ec96-41c3-95bf-a8624bee9e2e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.331143098Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1542024-11-01T16:42:33.357Z{"msg":"Crucible stats registered with UUID: fc446cdc-ec96-41c3-95bf-a8624bee9e2e","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.331181914Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1552024-11-01T16:42:33.388Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.365223405Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"task":"crutest"}
1562024-11-01T16:42:33.388Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.36527417Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220}
1572024-11-01T16:42:33.388Z{"msg":"fc446cdc-ec96-41c3-95bf-a8624bee9e2e active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.365310787Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1582024-11-01T16:42:33.392Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.365349413Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"client":"0","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1592024-11-01T16:42:33.392Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.365409857Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"client":"0","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1602024-11-01T16:42:33.392Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.365455852Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"client":"1","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1612024-11-01T16:42:33.393Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.365511727Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"client":"1","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1622024-11-01T16:42:33.393Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.365546233Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"client":"2","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1632024-11-01T16:42:33.393Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.365575821Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"client":"2","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1642024-11-01T16:42:33.393Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.365604708Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"":"io task","client":"2","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1652024-11-01T16:42:33.393Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.365633705Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"":"io task","client":"0","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1662024-11-01T16:42:33.393Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.365661452Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"":"io task","client":"1","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1672024-11-01T16:42:33.393Z{"msg":"ds_connection connected from Ok(127.0.0.1:36643)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.365690559Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"":"io task","client":"2","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1682024-11-01T16:42:33.393Z{"msg":"ds_connection connected from Ok(127.0.0.1:52178)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.365900218Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"":"io task","client":"1","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1692024-11-01T16:42:33.393Z{"msg":"ds_connection connected from Ok(127.0.0.1:56047)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.365931415Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"":"io task","client":"0","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1702024-11-01T16:42:33.393Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.366471142Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"client":"1","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1712024-11-01T16:42:33.393Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.366520547Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"client":"1","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1722024-11-01T16:42:33.393Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.366549464Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"client":"0","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1732024-11-01T16:42:33.393Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.36659264Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"client":"0","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1742024-11-01T16:42:33.393Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.366621027Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"client":"2","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1752024-11-01T16:42:33.393Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.366658693Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"client":"2","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1762024-11-01T16:42:33.393Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.367124557Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"client":"1","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1772024-11-01T16:42:33.393Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.367178342Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"client":"0","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1782024-11-01T16:42:33.394Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.367271073Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"client":"2","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1792024-11-01T16:42:33.394Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.367492021Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"client":"1","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1802024-11-01T16:42:33.394Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.367544866Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"client":"1","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1812024-11-01T16:42:33.395Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.367588201Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1822024-11-01T16:42:33.395Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.367629567Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"client":"0","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1832024-11-01T16:42:33.395Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.367669683Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"client":"0","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1842024-11-01T16:42:33.395Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.36770719Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1852024-11-01T16:42:33.395Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.367740786Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"client":"2","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1862024-11-01T16:42:33.395Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.367774403Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"client":"2","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1872024-11-01T16:42:33.395Z{"msg":"[0]R flush_numbers: [1693, 1693, 1693, 1693, 1693]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.367811369Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"":"downstairs","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1882024-11-01T16:42:33.395Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.367849096Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"":"downstairs","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1892024-11-01T16:42:33.395Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.367886182Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"":"downstairs","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1902024-11-01T16:42:33.395Z{"msg":"[1]R flush_numbers: [1693, 1693, 1693, 1693, 1693]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
1912024-11-01T16:42:33.395Z,"time":"2024-11-01T16:42:32.367924738Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"":"downstairs","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1922024-11-01T16:42:33.395Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.368045656Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"":"downstairs","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1932024-11-01T16:42:33.395Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.368082362Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"":"downstairs","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1942024-11-01T16:42:33.395Z{"msg":"[2]R flush_numbers: [1693, 1693, 1693, 1693, 1693]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.368117849Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"":"downstairs","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1952024-11-01T16:42:33.395Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.368140337Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"":"downstairs","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1962024-11-01T16:42:33.395Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.368172864Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"":"downstairs","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1972024-11-01T16:42:33.395Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"Region: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
1982024-11-01T16:42:33.398Ztime":"2024-11-01T16:42:32.36821074Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"":"downstairs","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
1992024-11-01T16:42:33.398Z{"msg":"Fill test
2002024-11-01T16:42:33.398ZGeneration requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.368256105Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"":"downstairs","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
2012024-11-01T16:42:33.398Z{"msg":"Next flush: 1694","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.368292512Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"":"downstairs","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
2022024-11-01T16:42:33.398Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.368324658Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"":"downstairs","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
2032024-11-01T16:42:33.398Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.368358495Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
2042024-11-01T16:42:33.398Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.368391622Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"client":"0","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
2052024-11-01T16:42:33.398Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.368425899Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"client":"1","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
2062024-11-01T16:42:33.398Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.368462175Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"client":"2","":"downstairs","session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
2072024-11-01T16:42:33.398Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.368494062Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
2082024-11-01T16:42:33.398Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.368521239Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
2092024-11-01T16:42:33.398Z{"msg":"fc446cdc-ec96-41c3-95bf-a8624bee9e2e is now active with session: d5214ddb-2451-4c7e-a963-a0e97d5decd6","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.368535937Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
2102024-11-01T16:42:33.398Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.368572174Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220}
2112024-11-01T16:42:33.398Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:32.368593122Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220,"session_id":"d5214ddb-2451-4c7e-a963-a0e97d5decd6"}
2122024-11-01T16:42:33.436ZRead and Verify all blocks (0..50 range:false)
2132024-11-01T16:42:33.479ZWrote out file "/var/tmp/test_up-build/verify_file"
2142024-11-01T16:42:33.479ZCLIENT: Tests done. All submitted work has been ACK'd
2152024-11-01T16:42:33.481Z----------------------------------------------------------------
2162024-11-01T16:42:33.481Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2172024-11-01T16:42:33.481ZStates: Active Active Active
2182024-11-01T16:42:33.481ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2192024-11-01T16:42:33.481Z 1002 Acked Read 50 Done Done Done false
2202024-11-01T16:42:33.481Z STATES DS:0 DS:1 DS:2 TOTAL
2212024-11-01T16:42:33.482Z Sent 0 0 0 0
2222024-11-01T16:42:33.482Z Done 1 1 1 3
2232024-11-01T16:42:33.482Z Skipped 0 0 0 0
2242024-11-01T16:42:33.482Z Error 0 0 0 0
2252024-11-01T16:42:33.482ZLast Flush: 1001 1001 1001
2262024-11-01T16:42:33.482ZDownstairs last five completed: 1001 1000
2272024-11-01T16:42:33.482ZUpstairs last five completed: 1002 1001 1000
2282024-11-01T16:42:33.482ZCLIENT: Up:0 ds:1 act:3
2292024-11-01T16:42:37.482Z----------------------------------------------------------------
2302024-11-01T16:42:37.482Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2312024-11-01T16:42:37.482ZDownstairs last five completed: 1003 1002 1001 1000
2322024-11-01T16:42:37.482ZUpstairs last five completed: 1003 1002 1001 1000
2332024-11-01T16:42:37.482ZCLIENT: Up:0 ds:0 act:3
2342024-11-01T16:42:37.482ZCLIENT: All crucible jobs finished, exiting program
2352024-11-01T16:42:37.485Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:42:36.45875551Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1220}
2362024-11-01T16:42:37.493ZRepair setup passed
2372024-11-01T16:42:37.497ZCopy the region for /var/tmp/test_up-build/8830
2382024-11-01T16:42:37.500Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2392024-11-01T16:42:37.502Z/var/tmp/bins/crutest fill -g 32 -q --key 91iihcUukm8TDD022wDf1OUglF26JChKuknMSR++tUY= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2402024-11-01T16:42:37.560Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.536421315Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"task":"crutest"}
2412024-11-01T16:42:37.563Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.537056112Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"task":"crutest"}
2422024-11-01T16:42:37.563Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.538853664Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"task":"crutest"}
2432024-11-01T16:42:37.563Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.539026287Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226}
2442024-11-01T16:42:37.563Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7796cc7e5b7a12d521784f7b6b2dd4b8dd6e6d5b\",\n git_commit_timestamp: \"2024-11-01T16:07:40.000000000Z\",\n git_branch: \"renovate/reqwest-0.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.539081931Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226}
2452024-11-01T16:42:37.563Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.539110959Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226}
2462024-11-01T16:42:37.563Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.539130097Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226}
2472024-11-01T16:42:37.566Z{"msg":"Crucible a12b9f7d-a3a9-4cec-b033-ce4872e37033 has session id: b0286ab4-a490-4d84-b70f-e444ebabe2a4","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.539594061Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2482024-11-01T16:42:37.566Z{"msg":"Upstairs opts: Upstairs UUID: a12b9f7d-a3a9-4cec-b033-ce4872e37033, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.539647195Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2492024-11-01T16:42:37.566Z{"msg":"Crucible stats registered with UUID: a12b9f7d-a3a9-4cec-b033-ce4872e37033","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.539672493Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2502024-11-01T16:42:37.597Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.573884478Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"task":"crutest"}
2512024-11-01T16:42:37.597Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.573944672Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226}
2522024-11-01T16:42:37.601Z{"msg":"a12b9f7d-a3a9-4cec-b033-ce4872e37033 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.573991857Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2532024-11-01T16:42:37.601Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.574041832Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"client":"0","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2542024-11-01T16:42:37.601Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.574150941Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"client":"0","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2552024-11-01T16:42:37.601Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.574213565Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"client":"1","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2562024-11-01T16:42:37.601Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.57426794Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"client":"1","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2572024-11-01T16:42:37.602Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.574581519Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"client":"2","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2582024-11-01T16:42:37.602Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.574638373Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"client":"2","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2592024-11-01T16:42:37.602Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.574689078Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"":"io task","client":"2","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2602024-11-01T16:42:37.602Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.574721625Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"":"io task","client":"0","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2612024-11-01T16:42:37.602Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.574749622Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"":"io task","client":"1","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2622024-11-01T16:42:37.602Z{"msg":"ds_connection connected from Ok(127.0.0.1:42577)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.574778059Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"":"io task","client":"2","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2632024-11-01T16:42:37.602Z{"msg":"ds_connection connected from Ok(127.0.0.1:51251)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.574812606Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"":"io task","client":"1","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2642024-11-01T16:42:37.602Z{"msg":"ds_connection connected from Ok(127.0.0.1:47892)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.574834684Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"":"io task","client":"0","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2652024-11-01T16:42:37.602Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.574949172Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"client":"1","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2662024-11-01T16:42:37.602Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.574978669Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"client":"1","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2672024-11-01T16:42:37.603Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.574993418Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"client":"0","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2682024-11-01T16:42:37.603Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.575006736Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"client":"0","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2692024-11-01T16:42:37.603Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.575019835Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"client":"2","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2702024-11-01T16:42:37.603Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.575042333Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"client":"2","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2712024-11-01T16:42:37.603Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.575647883Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"client":"0","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2722024-11-01T16:42:37.603Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.575805768Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"client":"1","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2732024-11-01T16:42:37.603Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.575891249Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"client":"2","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2742024-11-01T16:42:37.603Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.576056893Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"client":"0","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2752024-11-01T16:42:37.603Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.576094949Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"client":"0","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2762024-11-01T16:42:37.603Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.576111797Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2772024-11-01T16:42:37.603Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.576201058Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"client":"1","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2782024-11-01T16:42:37.603Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.576237595Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"client":"1","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2792024-11-01T16:42:37.603Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.576253523Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2802024-11-01T16:42:37.603Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.576266022Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"client":"2","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2812024-11-01T16:42:37.603Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.576280021Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"client":"2","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2822024-11-01T16:42:37.603Z{"msg":"[0]R flush_numbers: [1694, 1694, 1694, 1694, 1694]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.576301119Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"":"downstairs","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2832024-11-01T16:42:37.603Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.576315107Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"":"downstairs","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2842024-11-01T16:42:37.604Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.576328286Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"":"downstairs","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2852024-11-01T16:42:37.604Z{"msg":"[1]R flush_numbers: [1694, 1694, 1694, 1694, 1694]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.576348794Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"":"downstairs","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2862024-11-01T16:42:37.604Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.576362682Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"":"downstairs","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2872024-11-01T16:42:37.604Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.576375941Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"":"downstairs","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2882024-11-01T16:42:37.605Z{"msg":"[2]R flush_numbers: [1694, 1694, 1694, 1694, 1694]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.576395979Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"":"downstairs","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2892024-11-01T16:42:37.605Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.576409368Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"":"downstairs","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2902024-11-01T16:42:37.605Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.576422427Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"":"downstairs","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2912024-11-01T16:42:37.605Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.576442014Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"":"downstairs","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2922024-11-01T16:42:37.605Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.576455593Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"":"downstairs","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2932024-11-01T16:42:37.605ZWait for a query_work_queue command to finish before sending IO
2942024-11-01T16:42:37.605Z{"msg":"Next flush: 1695","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.576479521Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"":"downstairs","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2952024-11-01T16:42:37.605Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.576499729Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"":"downstairs","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2962024-11-01T16:42:37.605Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.576533116Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2972024-11-01T16:42:37.605Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.576546694Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"client":"0","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2982024-11-01T16:42:37.605Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.576559493Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"client":"1","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
2992024-11-01T16:42:37.605Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.576579501Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"client":"2","":"downstairs","session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
3002024-11-01T16:42:37.605Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.57659305Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
3012024-11-01T16:42:37.606Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.576605628Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
3022024-11-01T16:42:37.606Z{"msg":"a12b9f7d-a3a9-4cec-b033-ce4872e37033 is now active with session: b0286ab4-a490-4d84-b70f-e444ebabe2a4","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.576624956Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
3032024-11-01T16:42:37.606Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.576637945Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226,"session_id":"b0286ab4-a490-4d84-b70f-e444ebabe2a4"}
3042024-11-01T16:42:37.606Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:36.576650524Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226}
3052024-11-01T16:42:37.606ZRegion: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
3062024-11-01T16:42:37.610ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
3072024-11-01T16:42:37.610ZFill test
3082024-11-01T16:42:37.645ZRead and Verify all blocks (0..50 range:false)
3092024-11-01T16:42:37.684ZWrote out file "/var/tmp/test_up-build/verify_file"
3102024-11-01T16:42:37.684ZCLIENT: Tests done. All submitted work has been ACK'd
3112024-11-01T16:42:37.688Z----------------------------------------------------------------
3122024-11-01T16:42:37.688Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
3132024-11-01T16:42:37.688ZStates: Active Active Active
3142024-11-01T16:42:37.688ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
3152024-11-01T16:42:37.688Z 1002 Acked Read 50 Done Sent Sent false
3162024-11-01T16:42:37.688Z STATES DS:0 DS:1 DS:2 TOTAL
3172024-11-01T16:42:37.688Z Sent 0 1 1 2
3182024-11-01T16:42:37.688Z Done 1 0 0 1
3192024-11-01T16:42:37.688Z Skipped 0 0 0 0
3202024-11-01T16:42:37.688Z Error 0 0 0 0
3212024-11-01T16:42:37.688ZLast Flush: 1001 1001 1001
3222024-11-01T16:42:37.688ZDownstairs last five completed: 1001 1000
3232024-11-01T16:42:37.688ZUpstairs last five completed: 1002 1001 1000
3242024-11-01T16:42:37.689ZCLIENT: Up:0 ds:1 act:3
3252024-11-01T16:42:41.687Z----------------------------------------------------------------
3262024-11-01T16:42:41.687Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
3272024-11-01T16:42:41.687ZDownstairs last five completed: 1003 1002 1001 1000
3282024-11-01T16:42:41.687ZUpstairs last five completed: 1003 1002 1001 1000
3292024-11-01T16:42:41.687ZCLIENT: Up:0 ds:0 act:3
3302024-11-01T16:42:41.688ZCLIENT: All crucible jobs finished, exiting program
3312024-11-01T16:42:41.690Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:42:40.662434646Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1226}
3322024-11-01T16:42:41.695ZRepair part 1 passed
3332024-11-01T16:42:41.695Z
3342024-11-01T16:42:41.695ZKill the current downstairs
3352024-11-01T16:42:41.861ZDownstairs 2 stopped
3362024-11-01T16:42:41.861Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
3372024-11-01T16:42:41.863ZNow put back the original so we have a mismatch
3382024-11-01T16:42:41.864Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
3392024-11-01T16:42:41.866ZRestart downstairs with old directory
3402024-11-01T16:42:41.922Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
3412024-11-01T16:42:41.937ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3422024-11-01T16:42:41.942Z 0 000-009 32 32 31 1695 1695 1694 F F F <---
3432024-11-01T16:42:41.942Z 1 010-019 32 32 31 1695 1695 1694 F F F <---
3442024-11-01T16:42:41.942Z 2 020-029 32 32 31 1695 1695 1694 F F F <---
3452024-11-01T16:42:41.942Z 3 030-039 32 32 31 1695 1695 1694 F F F <---
3462024-11-01T16:42:41.942Z 4 040-049 32 32 31 1695 1695 1694 F F F <---
3472024-11-01T16:42:41.942ZMax gen: 32, Max flush: 1695
3482024-11-01T16:42:41.942ZError: Difference in extent metadata found!
3492024-11-01T16:42:41.942Zdump test found error as expected
3502024-11-01T16:42:41.942Z
3512024-11-01T16:42:41.942Z
3522024-11-01T16:42:41.942Z/var/tmp/bins/crutest verify --range -g 33 -q --key 91iihcUukm8TDD022wDf1OUglF26JChKuknMSR++tUY= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
3532024-11-01T16:42:41.996Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:40.971964987Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"task":"crutest"}
3542024-11-01T16:42:41.999Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:40.972584365Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"task":"crutest"}
3552024-11-01T16:42:41.999Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:40.974391857Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"task":"crutest"}
3562024-11-01T16:42:41.999Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:40.97455934Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234}
3572024-11-01T16:42:41.999Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7796cc7e5b7a12d521784f7b6b2dd4b8dd6e6d5b\",\n git_commit_timestamp: \"2024-11-01T16:07:40.000000000Z\",\n git_branch: \"renovate/reqwest-0.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:40.974601626Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234}
3582024-11-01T16:42:41.999Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:40.974623854Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234}
3592024-11-01T16:42:41.999Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:40.974639992Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234}
3602024-11-01T16:42:42.002Z{"msg":"Crucible e79cb41a-43c2-47c1-b604-5bd0a6040cca has session id: 55091d4b-25b4-43ed-9b3c-6504fb58a2d0","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:40.975083568Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3612024-11-01T16:42:42.002Z{"msg":"Upstairs opts: Upstairs UUID: e79cb41a-43c2-47c1-b604-5bd0a6040cca, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:40.975136813Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3622024-11-01T16:42:42.002Z{"msg":"Crucible stats registered with UUID: e79cb41a-43c2-47c1-b604-5bd0a6040cca","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:40.975174249Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3632024-11-01T16:42:42.034Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:41.009566746Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"task":"crutest"}
3642024-11-01T16:42:42.034Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:41.009634329Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234}
3652024-11-01T16:42:42.036Z{"msg":"e79cb41a-43c2-47c1-b604-5bd0a6040cca active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:41.009676645Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3662024-11-01T16:42:42.036Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:41.00972441Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"client":"0","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3672024-11-01T16:42:42.037Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:41.009797033Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"client":"0","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3682024-11-01T16:42:42.037Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:41.009836749Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"client":"1","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3692024-11-01T16:42:42.037Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:41.009891944Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"client":"1","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3702024-11-01T16:42:42.037Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:41.009923021Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"client":"2","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3712024-11-01T16:42:42.037Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:41.009972086Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"client":"2","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3722024-11-01T16:42:42.037Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:41.010005113Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"":"io task","client":"0","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3732024-11-01T16:42:42.037Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:41.010040089Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"":"io task","client":"2","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3742024-11-01T16:42:42.037Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:41.010082435Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"":"io task","client":"1","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3752024-11-01T16:42:42.037Z{"msg":"ds_connection connected from Ok(127.0.0.1:60144)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:41.010114322Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"":"io task","client":"0","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3762024-11-01T16:42:42.037Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:42:41.010159417Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"":"io task","client":"2","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3772024-11-01T16:42:42.038Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:42:41.010245839Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"":"io task","client":"2","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3782024-11-01T16:42:42.038Z{"msg":"ds_connection connected from Ok(127.0.0.1:52235)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:41.010283495Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"":"io task","client":"1","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3792024-11-01T16:42:42.038Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:42:41.010300453Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3802024-11-01T16:42:42.038Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:41.010313502Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"client":"2","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3812024-11-01T16:42:42.038Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:41.010326741Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"client":"2","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3822024-11-01T16:42:42.038Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:41.010455548Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"":"io task","client":"2","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3832024-11-01T16:42:42.038Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:41.010881236Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"client":"0","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3842024-11-01T16:42:42.038Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:41.010924372Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"client":"0","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3852024-11-01T16:42:42.038Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:41.010982886Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"client":"1","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3862024-11-01T16:42:42.038Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:41.011026221Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"client":"1","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3872024-11-01T16:42:42.039Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:41.01154895Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"client":"1","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3882024-11-01T16:42:42.039Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:41.011602055Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"client":"0","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3892024-11-01T16:42:42.039Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:41.011967439Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"client":"0","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3902024-11-01T16:42:42.039Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:41.012008004Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"client":"0","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3912024-11-01T16:42:42.039Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:41.012023663Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3922024-11-01T16:42:42.039Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:41.012036652Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"client":"1","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3932024-11-01T16:42:42.039Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:41.012057559Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"client":"1","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3942024-11-01T16:42:42.039Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:41.012071478Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3952024-11-01T16:42:52.037Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.011171841Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"":"io task","client":"2","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3962024-11-01T16:42:52.042Z{"msg":"ds_connection connected from Ok(127.0.0.1:33606)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.011449204Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"":"io task","client":"2","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3972024-11-01T16:42:52.042Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.012286791Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"client":"2","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3982024-11-01T16:42:52.042Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.012329347Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"client":"2","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
3992024-11-01T16:42:52.042Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.012871343Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"client":"2","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4002024-11-01T16:42:52.042Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.013228128Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"client":"2","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4012024-11-01T16:42:52.042Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.013269624Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"client":"2","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4022024-11-01T16:42:52.042Z{"msg":"[0]R flush_numbers: [1695, 1695, 1695, 1695, 1695]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.013290402Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4032024-11-01T16:42:52.042Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.013317789Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4042024-11-01T16:42:52.042Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.013336197Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4052024-11-01T16:42:52.042Z{"msg":"[1]R flush_numbers: [1695, 1695, 1695, 1695, 1695]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.013353025Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4062024-11-01T16:42:52.042Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.013377743Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4072024-11-01T16:42:52.043Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.013391582Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4082024-11-01T16:42:52.043Z{"msg":"[2]R flush_numbers: [1694, 1694, 1694, 1694, 1694]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.01341209Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4092024-11-01T16:42:52.043Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.013426478Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4102024-11-01T16:42:52.043Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.013446756Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4112024-11-01T16:42:52.043Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.013460365Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4122024-11-01T16:42:52.043Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.013480303Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4132024-11-01T16:42:52.043Z{"msg":"Next flush: 1696","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.013493991Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4142024-11-01T16:42:52.043Z{"msg":"Extent 0 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.01350709Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4152024-11-01T16:42:52.043Z{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.013527398Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4162024-11-01T16:42:52.043Z{"msg":"extent:0 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.013542037Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4172024-11-01T16:42:52.043Z{"msg":"extent:0 flush: 1695 1695 1694 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.013563425Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4182024-11-01T16:42:52.043Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.013578153Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4192024-11-01T16:42:52.043Z{"msg":"extent:0 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.013598711Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4202024-11-01T16:42:52.043Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.0136135Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4212024-11-01T16:42:52.043Z{"msg":"find dest for source 0 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.013634388Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4222024-11-01T16:42:52.044Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.013648696Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4232024-11-01T16:42:52.044Z{"msg":"Extent 1 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.013686763Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4242024-11-01T16:42:52.044Z{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.0137092Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4252024-11-01T16:42:52.044Z{"msg":"extent:1 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.013724049Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4262024-11-01T16:42:52.044Z{"msg":"extent:1 flush: 1695 1695 1694 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.013745327Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4272024-11-01T16:42:52.045Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.013760065Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4282024-11-01T16:42:52.045Z{"msg":"extent:1 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.013782313Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4292024-11-01T16:42:52.045Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.013797281Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4302024-11-01T16:42:52.045Z{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.013818419Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4312024-11-01T16:42:52.045Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.013832778Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4322024-11-01T16:42:52.045Z{"msg":"Extent 2 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.013854086Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4332024-11-01T16:42:52.045Z{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.013873194Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4342024-11-01T16:42:52.045Z{"msg":"extent:2 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.01391105Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4352024-11-01T16:42:52.045Z{"msg":"extent:2 flush: 1695 1695 1694 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.013950786Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4362024-11-01T16:42:52.045Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.013978494Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4372024-11-01T16:42:52.045Z{"msg":"extent:2 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.01401399Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4382024-11-01T16:42:52.045Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.014032228Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4392024-11-01T16:42:52.045Z{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.014047497Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4402024-11-01T16:42:52.045Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.014069754Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4412024-11-01T16:42:52.045Z{"msg":"Extent 3 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.014085743Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4422024-11-01T16:42:52.045Z{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.014099192Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4432024-11-01T16:42:52.045Z{"msg":"extent:3 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.014120669Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4442024-11-01T16:42:52.045Z{"msg":"extent:3 flush: 1695 1695 1694 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.014135678Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4452024-11-01T16:42:52.045Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.014149827Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4462024-11-01T16:42:52.045Z{"msg":"extent:3 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.014170904Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4472024-11-01T16:42:52.045Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.014185823Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4482024-11-01T16:42:52.045Z{"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.014209041Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4492024-11-01T16:42:52.048Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.014232189Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4502024-11-01T16:42:52.048Z{"msg":"Extent 4 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.014269295Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4512024-11-01T16:42:52.048Z{"msg":"First source client ID for extent 4","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.014287093Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4522024-11-01T16:42:52.048Z{"msg":"extent:4 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.014301232Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4532024-11-01T16:42:52.048Z{"msg":"extent:4 flush: 1695 1695 1694 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.014323959Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4542024-11-01T16:42:52.048Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.014338878Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4552024-11-01T16:42:52.048Z{"msg":"extent:4 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.014353077Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4562024-11-01T16:42:52.048Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.014368385Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4572024-11-01T16:42:52.049Z{"msg":"find dest for source 0 for extent at index 4","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.014384083Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4582024-11-01T16:42:52.049Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.014404561Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4592024-11-01T16:42:52.049Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.014440658Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"client":"0","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4602024-11-01T16:42:52.049Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.014467795Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"client":"1","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4612024-11-01T16:42:52.049Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.014502092Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"client":"2","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4622024-11-01T16:42:52.049Z{"msg":"starting reconciliation 161622fc-ea57-4d4a-aff3-ee3ad6ac4f75: found 5 extents that need repair","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.01452223Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4632024-11-01T16:42:52.049Z{"msg":"Full repair list: {ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.014557986Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4642024-11-01T16:42:52.049Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 1696, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 1696, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 1696, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 1696, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 1696, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.014666006Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4652024-11-01T16:42:52.049Z{"msg":"no Nexus client from DNS, aborting start notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:42:51.014702362Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"reconcile":"161622fc-ea57-4d4a-aff3-ee3ad6ac4f75","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4662024-11-01T16:42:52.049Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:42:51.014730919Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"reconcile":"161622fc-ea57-4d4a-aff3-ee3ad6ac4f75","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4672024-11-01T16:42:52.049Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:42:51.014757286Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"reconcile":"161622fc-ea57-4d4a-aff3-ee3ad6ac4f75","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4682024-11-01T16:42:52.049Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.014783504Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"client":"2","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4692024-11-01T16:42:52.049Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:42:51.01481885Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"reconcile":"161622fc-ea57-4d4a-aff3-ee3ad6ac4f75","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4702024-11-01T16:42:52.054Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:42:51.028094387Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"reconcile":"161622fc-ea57-4d4a-aff3-ee3ad6ac4f75","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4712024-11-01T16:42:52.056Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:42:51.028498147Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"reconcile":"161622fc-ea57-4d4a-aff3-ee3ad6ac4f75","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4722024-11-01T16:42:52.056Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:42:51.028787508Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"reconcile":"161622fc-ea57-4d4a-aff3-ee3ad6ac4f75","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4732024-11-01T16:42:52.056Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.029060901Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"client":"2","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4742024-11-01T16:42:52.056Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:42:51.029110826Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"reconcile":"161622fc-ea57-4d4a-aff3-ee3ad6ac4f75","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4752024-11-01T16:42:52.063Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:42:51.037551661Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"reconcile":"161622fc-ea57-4d4a-aff3-ee3ad6ac4f75","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4762024-11-01T16:42:52.066Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:42:51.037903506Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"reconcile":"161622fc-ea57-4d4a-aff3-ee3ad6ac4f75","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4772024-11-01T16:42:52.066Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:42:51.038238193Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"reconcile":"161622fc-ea57-4d4a-aff3-ee3ad6ac4f75","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4782024-11-01T16:42:52.066Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.038527944Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"client":"2","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4792024-11-01T16:42:52.066Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:42:51.03857072Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"reconcile":"161622fc-ea57-4d4a-aff3-ee3ad6ac4f75","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4802024-11-01T16:42:52.073Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:42:51.047410006Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"reconcile":"161622fc-ea57-4d4a-aff3-ee3ad6ac4f75","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4812024-11-01T16:42:52.075Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:42:51.047817465Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"reconcile":"161622fc-ea57-4d4a-aff3-ee3ad6ac4f75","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4822024-11-01T16:42:52.075Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:42:51.048047612Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"reconcile":"161622fc-ea57-4d4a-aff3-ee3ad6ac4f75","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4832024-11-01T16:42:52.075Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.048288408Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"client":"2","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4842024-11-01T16:42:52.076Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:42:51.048334184Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"reconcile":"161622fc-ea57-4d4a-aff3-ee3ad6ac4f75","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4852024-11-01T16:42:52.083Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:42:51.056786408Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"reconcile":"161622fc-ea57-4d4a-aff3-ee3ad6ac4f75","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4862024-11-01T16:42:52.085Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:42:51.057543073Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"reconcile":"161622fc-ea57-4d4a-aff3-ee3ad6ac4f75","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4872024-11-01T16:42:52.085Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:42:51.057807957Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"reconcile":"161622fc-ea57-4d4a-aff3-ee3ad6ac4f75","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4882024-11-01T16:42:52.085Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.05807465Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"client":"2","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4892024-11-01T16:42:52.085Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:42:51.058115316Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"reconcile":"161622fc-ea57-4d4a-aff3-ee3ad6ac4f75","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4902024-11-01T16:42:52.092Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:42:51.066453041Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"reconcile":"161622fc-ea57-4d4a-aff3-ee3ad6ac4f75","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4912024-11-01T16:42:52.094Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.066784498Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4922024-11-01T16:42:52.094Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.066818645Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"client":"0","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4932024-11-01T16:42:52.094Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.066841123Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"client":"1","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4942024-11-01T16:42:52.094Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.066880989Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"client":"2","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4952024-11-01T16:42:52.095Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.066904106Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4962024-11-01T16:42:52.095ZWait for a query_work_queue command to finish before sending IO
4972024-11-01T16:42:52.095Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.066948972Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4982024-11-01T16:42:52.095Z{"msg":"e79cb41a-43c2-47c1-b604-5bd0a6040cca is now active with session: 55091d4b-25b4-43ed-9b3c-6504fb58a2d0","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.066987178Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
4992024-11-01T16:42:52.095Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:42:51.067013526Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234}
5002024-11-01T16:42:52.095Z{"msg":"no Nexus client from DNS, aborting finish notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:42:51.067045732Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234,"reconcile":"161622fc-ea57-4d4a-aff3-ee3ad6ac4f75","":"downstairs","":"downstairs","session_id":"55091d4b-25b4-43ed-9b3c-6504fb58a2d0"}
5012024-11-01T16:42:52.095ZRegion: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
5022024-11-01T16:42:52.095ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
5032024-11-01T16:42:52.095ZRead and Verify all blocks (0..50 range:true)
5042024-11-01T16:42:52.142ZWrote out file "/var/tmp/test_up-build/verify_file"
5052024-11-01T16:42:52.142ZVerify test completed
5062024-11-01T16:42:52.145ZWrote out file "/var/tmp/test_up-build/verify_file"
5072024-11-01T16:42:52.145ZCLIENT: Tests done. All submitted work has been ACK'd
5082024-11-01T16:42:52.145Z----------------------------------------------------------------
5092024-11-01T16:42:52.145Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
5102024-11-01T16:42:52.145ZStates: Active Active Active
5112024-11-01T16:42:52.145ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
5122024-11-01T16:42:52.145Z 1000 Acked Read 50 Done Done Done false
5132024-11-01T16:42:52.145Z STATES DS:0 DS:1 DS:2 TOTAL
5142024-11-01T16:42:52.145Z Sent 0 0 0 0
5152024-11-01T16:42:52.145Z Done 1 1 1 3
5162024-11-01T16:42:52.145Z Skipped 0 0 0 0
5172024-11-01T16:42:52.145Z Error 0 0 0 0
5182024-11-01T16:42:52.145ZLast Flush: 0 0 0
5192024-11-01T16:42:52.145ZDownstairs last five completed:
5202024-11-01T16:42:52.145ZUpstairs last five completed: 1000
5212024-11-01T16:42:52.145ZCLIENT: Up:0 ds:1 act:3
5222024-11-01T16:42:56.146Z----------------------------------------------------------------
5232024-11-01T16:42:56.146Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
5242024-11-01T16:42:56.146ZDownstairs last five completed: 1001 1000
5252024-11-01T16:42:56.146ZUpstairs last five completed: 1001 1000
5262024-11-01T16:42:56.146ZCLIENT: Up:0 ds:0 act:3
5272024-11-01T16:42:56.146ZCLIENT: All crucible jobs finished, exiting program
5282024-11-01T16:42:56.149Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:42:55.119040218Z","hostname":"w-01JBM9EHP08T9WT7D3HWEXQS78","pid":1234}
5292024-11-01T16:42:56.153ZRepair part 2 passed
5302024-11-01T16:42:56.155Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
5312024-11-01T16:42:56.170ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
5322024-11-01T16:42:56.170Z 0 000-009 32 32 32 1695 1695 1695 F F F
5332024-11-01T16:42:56.170Z 1 010-019 32 32 32 1695 1695 1695 F F F
5342024-11-01T16:42:56.170Z 2 020-029 32 32 32 1695 1695 1695 F F F
5352024-11-01T16:42:56.170Z 3 030-039 32 32 32 1695 1695 1695 F F F
5362024-11-01T16:42:56.170Z 4 040-049 32 32 32 1695 1695 1695 F F F
5372024-11-01T16:42:56.173ZMax gen: 32, Max flush: 1695
5382024-11-01T16:42:56.175Zdump test passed
5392024-11-01T16:42:56.175Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
5402024-11-01T16:42:56.188Z Extent 1
5412024-11-01T16:42:56.191ZGEN 32 32 32
5422024-11-01T16:42:56.191ZFLUSH_ID 1695 1695 1695
5432024-11-01T16:42:56.191ZDIRTY
5442024-11-01T16:42:56.191Z
5452024-11-01T16:42:56.191ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
5462024-11-01T16:42:56.191Z010 A A A A A A
5472024-11-01T16:42:56.193Z011 A A A A A A
5482024-11-01T16:42:56.193Z012 A A A A A A
5492024-11-01T16:42:56.193Z013 A A A A A A
5502024-11-01T16:42:56.196Z014 A A A A A A
5512024-11-01T16:42:56.196Z015 A A A A A A
5522024-11-01T16:42:56.198Z016 A A A A A A
5532024-11-01T16:42:56.198Z017 A A A A A A
5542024-11-01T16:42:56.201Z018 A A A A A A
5552024-11-01T16:42:56.201Z019 A A A A A A
5562024-11-01T16:42:56.204Zdump extent test passed
5572024-11-01T16:42:56.204Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
5582024-11-01T16:42:56.218ZExtent 2 Block in extent 0 Actual block 20
5592024-11-01T16:42:56.218Z
5602024-11-01T16:42:56.221Z DATA SHA256 VER
5612024-11-01T16:42:56.221Z------ ---------------------------------------------------------------- ---
5622024-11-01T16:42:56.221Z 0 aaddc8d6a2f8f2e98699a0ab91e5dfba796b5a7c4ee513cfef24c3c408d1fc78 A
5632024-11-01T16:42:56.221Z 1 aaddc8d6a2f8f2e98699a0ab91e5dfba796b5a7c4ee513cfef24c3c408d1fc78 A
5642024-11-01T16:42:56.221Z 2 aaddc8d6a2f8f2e98699a0ab91e5dfba796b5a7c4ee513cfef24c3c408d1fc78 A
5652024-11-01T16:42:56.221Z
5662024-11-01T16:42:56.221ZNONCES 0 1 2 DIFF
5672024-11-01T16:42:56.221Z------ ------------------------ ------------------------ ------------------------ -----
5682024-11-01T16:42:56.224Z 0 bc7cd2c45161ad0e9593423d bc7cd2c45161ad0e9593423d bc7cd2c45161ad0e9593423d
5692024-11-01T16:42:56.224Z
5702024-11-01T16:42:56.224Z TAGS 0 1 2 DIFF
5712024-11-01T16:42:56.224Z------ -------------------------------- -------------------------------- -------------------------------- -----
5722024-11-01T16:42:56.224Z 0 c62ce4f8b19168ef62c430db15f3bc89 c62ce4f8b19168ef62c430db15f3bc89 c62ce4f8b19168ef62c430db15f3bc89
5732024-11-01T16:42:56.224Z
5742024-11-01T16:42:56.224ZHASHES 0 1 2 DIFF
5752024-11-01T16:42:56.224Z------ ---------------- ---------------- ---------------- -----
5762024-11-01T16:42:56.224Z
5772024-11-01T16:42:56.226Zdump block test passed
5782024-11-01T16:42:56.226ZInitial upstairs tests have completed, stopping all downstairs
5792024-11-01T16:43:01.287ZCreating 4 larger downstairs regions
5802024-11-01T16:43:01.832ZStarting 4 downstairs
5812024-11-01T16:43:06.839Zdsc restarted at PID: 1247
5822024-11-01T16:43:06.951ZDownstairs client 1 not running, waiting for it
5832024-11-01T16:43:12.011ZDownstairs client 1 not running, waiting for it
5842024-11-01T16:43:17.072ZDownstairs client 1 not running, waiting for it
5852024-11-01T16:43:22.132ZDownstairs client 1 not running, waiting for it
5862024-11-01T16:43:27.193ZDownstairs client 1 not running, waiting for it
5872024-11-01T16:43:32.253ZDownstairs client 1 not running, waiting for it
5882024-11-01T16:43:37.314ZDownstairs client 1 not running, waiting for it
5892024-11-01T16:43:42.374ZDownstairs client 1 not running, waiting for it
5902024-11-01T16:43:47.435ZDownstairs client 1 not running, waiting for it
5912024-11-01T16:43:52.494ZDownstairs client 1 not running, waiting for it
5922024-11-01T16:43:57.555ZDownstairs client 1 not running, waiting for it
5932024-11-01T16:44:02.615ZDownstairs client 1 not running, waiting for it
5942024-11-01T16:44:07.675ZDownstairs client 1 not running, waiting for it
5952024-11-01T16:44:12.736ZDownstairs client 1 not running, waiting for it
5962024-11-01T16:44:17.796ZDownstairs client 1 not running, waiting for it
5972024-11-01T16:44:22.856ZDownstairs client 1 not running, waiting for it
5982024-11-01T16:44:27.915ZDownstairs client 1 not running, waiting for it
5992024-11-01T16:44:32.976ZDownstairs client 1 not running, waiting for it
6002024-11-01T16:44:38.037ZDownstairs client 1 not running, waiting for it
6012024-11-01T16:44:43.098ZDownstairs client 1 not running, waiting for it
6022024-11-01T16:44:48.158ZDownstairs client 1 not running, waiting for it
6032024-11-01T16:44:53.218ZDownstairs client 1 not running, waiting for it
6042024-11-01T16:44:58.281ZDownstairs client 1 not running, waiting for it
6052024-11-01T16:45:03.341ZDownstairs client 1 not running, waiting for it
6062024-11-01T16:45:08.403ZDownstairs client 1 not running, waiting for it
6072024-11-01T16:45:13.463ZDownstairs client 1 not running, waiting for it
6082024-11-01T16:45:18.523ZDownstairs client 1 not running, waiting for it
6092024-11-01T16:45:23.582ZDownstairs client 1 not running, waiting for it
6102024-11-01T16:45:28.643ZDownstairs client 1 not running, waiting for it
6112024-11-01T16:45:33.703ZDownstairs client 1 not running, waiting for it
6122024-11-01T16:45:38.764ZDownstairs client 1 not running, waiting for it
6132024-11-01T16:45:43.825ZDownstairs client 1 not running, waiting for it
6142024-11-01T16:45:48.885ZDownstairs client 1 not running, waiting for it
6152024-11-01T16:45:53.946ZDownstairs client 1 not running, waiting for it
6162024-11-01T16:45:59.007ZDownstairs client 1 not running, waiting for it
6172024-11-01T16:46:04.088ZDownstairs client 1 not running, waiting for it
6182024-11-01T16:46:09.134ZDownstairs client 1 not running, waiting for it
6192024-11-01T16:46:14.193ZDownstairs client 1 not running, waiting for it
6202024-11-01T16:46:19.250ZDownstairs client 1 not running, waiting for it
6212024-11-01T16:46:24.313ZDownstairs client 1 not running, waiting for it
6222024-11-01T16:46:29.372ZDownstairs client 1 not running, waiting for it
6232024-11-01T16:46:34.433ZDownstairs client 1 not running, waiting for it
6242024-11-01T16:46:39.494ZDownstairs client 1 not running, waiting for it
6252024-11-01T16:46:44.555ZDownstairs client 1 not running, waiting for it
6262024-11-01T16:46:49.616ZDownstairs client 1 not running, waiting for it
6272024-11-01T16:46:54.680ZDownstairs client 1 not running, waiting for it
6282024-11-01T16:46:59.739ZDownstairs client 1 not running, waiting for it
6292024-11-01T16:47:04.799ZDownstairs client 1 not running, waiting for it
6302024-11-01T16:47:09.860ZDownstairs client 1 not running, waiting for it
6312024-11-01T16:47:14.921ZDownstairs client 1 not running, waiting for it
6322024-11-01T16:47:19.981ZDownstairs client 1 not running, waiting for it
6332024-11-01T16:47:25.042ZDownstairs client 1 not running, waiting for it
6342024-11-01T16:47:30.103ZDownstairs client 1 not running, waiting for it
6352024-11-01T16:47:35.164ZDownstairs client 1 not running, waiting for it
6362024-11-01T16:47:40.225ZDownstairs client 1 not running, waiting for it
6372024-11-01T16:47:45.286ZDownstairs client 1 not running, waiting for it
6382024-11-01T16:47:50.347ZDownstairs client 1 not running, waiting for it
6392024-11-01T16:47:55.409ZDownstairs client 1 not running, waiting for it
6402024-11-01T16:48:00.469ZDownstairs client 1 not running, waiting for it
6412024-11-01T16:48:05.529ZDownstairs client 1 not running, waiting for it
6422024-11-01T16:48:10.590ZDownstairs client 1 not running, waiting for it
6432024-11-01T16:48:15.651ZDownstairs client 1 not running, waiting for it
6442024-11-01T16:48:20.712ZDownstairs client 1 not running, waiting for it
6452024-11-01T16:48:25.774ZDownstairs client 1 not running, waiting for it
6462024-11-01T16:48:30.834ZDownstairs client 1 not running, waiting for it
6472024-11-01T16:48:35.894ZDownstairs client 1 not running, waiting for it
6482024-11-01T16:48:40.956ZDownstairs client 1 not running, waiting for it
6492024-11-01T16:48:46.017ZDownstairs client 1 not running, waiting for it
6502024-11-01T16:48:51.077ZDownstairs client 1 not running, waiting for it
6512024-11-01T16:48:56.138ZDownstairs client 1 not running, waiting for it
6522024-11-01T16:49:01.198ZDownstairs client 1 not running, waiting for it
6532024-11-01T16:49:06.257ZDownstairs client 1 not running, waiting for it
6542024-11-01T16:49:11.317ZDownstairs client 1 not running, waiting for it
6552024-11-01T16:49:16.378ZDownstairs client 1 not running, waiting for it
6562024-11-01T16:49:21.438ZDownstairs client 1 not running, waiting for it
6572024-11-01T16:49:26.501ZDownstairs client 1 not running, waiting for it
6582024-11-01T16:49:31.558ZDownstairs client 1 not running, waiting for it
6592024-11-01T16:49:36.618ZDownstairs client 1 not running, waiting for it
6602024-11-01T16:49:41.680ZDownstairs client 1 not running, waiting for it
6612024-11-01T16:49:46.740ZDownstairs client 1 not running, waiting for it
6622024-11-01T16:49:51.800ZDownstairs client 1 not running, waiting for it
6632024-11-01T16:49:56.860ZDownstairs client 1 not running, waiting for it
6642024-11-01T16:50:01.920ZDownstairs client 1 not running, waiting for it
6652024-11-01T16:50:06.982ZDownstairs client 1 not running, waiting for it
6662024-11-01T16:50:12.082ZDownstairs client 1 not running, waiting for it
6672024-11-01T16:50:17.104ZDownstairs client 1 not running, waiting for it
6682024-11-01T16:50:22.163ZDownstairs client 1 not running, waiting for it
6692024-11-01T16:50:27.224ZDownstairs client 1 not running, waiting for it
6702024-11-01T16:50:32.284ZDownstairs client 1 not running, waiting for it
6712024-11-01T16:50:37.345ZDownstairs client 1 not running, waiting for it
6722024-11-01T16:50:42.406ZDownstairs client 1 not running, waiting for it
6732024-11-01T16:50:47.466ZDownstairs client 1 not running, waiting for it
6742024-11-01T16:50:52.527ZDownstairs client 1 not running, waiting for it
6752024-11-01T16:50:57.587ZDownstairs client 1 not running, waiting for it
6762024-11-01T16:51:02.649ZDownstairs client 1 not running, waiting for it
6772024-11-01T16:51:07.708ZDownstairs client 1 not running, waiting for it
6782024-11-01T16:51:12.770ZDownstairs client 1 not running, waiting for it
6792024-11-01T16:51:17.830ZDownstairs client 1 not running, waiting for it
6802024-11-01T16:51:22.889ZDownstairs client 1 not running, waiting for it
6812024-11-01T16:51:27.949ZDownstairs client 1 not running, waiting for it
6822024-11-01T16:51:33.009ZDownstairs client 1 not running, waiting for it
6832024-11-01T16:51:38.069ZDownstairs client 1 not running, waiting for it
6842024-11-01T16:51:43.130ZDownstairs client 1 not running, waiting for it
6852024-11-01T16:51:48.193ZDownstairs client 1 not running, waiting for it
6862024-11-01T16:51:53.255ZDownstairs client 1 not running, waiting for it
6872024-11-01T16:51:58.314ZDownstairs client 1 not running, waiting for it
6882024-11-01T16:52:03.375ZDownstairs client 1 not running, waiting for it
6892024-11-01T16:52:08.437ZDownstairs client 1 not running, waiting for it
6902024-11-01T16:52:13.497ZDownstairs client 1 not running, waiting for it
6912024-11-01T16:52:18.558ZDownstairs client 1 not running, waiting for it
6922024-11-01T16:52:23.619ZDownstairs client 1 not running, waiting for it
6932024-11-01T16:52:28.680ZDownstairs client 1 not running, waiting for it
6942024-11-01T16:52:33.742ZDownstairs client 1 not running, waiting for it
6952024-11-01T16:52:38.801ZDownstairs client 1 not running, waiting for it
6962024-11-01T16:52:43.863ZDownstairs client 1 not running, waiting for it
6972024-11-01T16:52:48.924ZDownstairs client 1 not running, waiting for it
6982024-11-01T16:52:53.983ZDownstairs client 1 not running, waiting for it
6992024-11-01T16:52:59.044ZDownstairs client 1 not running, waiting for it
7002024-11-01T16:53:04.104ZDownstairs client 1 not running, waiting for it
7012024-11-01T16:53:09.166ZDownstairs client 1 not running, waiting for it
7022024-11-01T16:53:14.226ZDownstairs client 1 not running, waiting for it
7032024-11-01T16:53:19.287ZDownstairs client 1 not running, waiting for it
7042024-11-01T16:53:24.348ZDownstairs client 1 not running, waiting for it
7052024-11-01T16:53:29.409ZDownstairs client 1 not running, waiting for it
7062024-11-01T16:53:34.469ZDownstairs client 1 not running, waiting for it
7072024-11-01T16:53:39.532ZDownstairs client 1 not running, waiting for it
7082024-11-01T16:53:44.592ZDownstairs client 1 not running, waiting for it
7092024-11-01T16:53:49.652ZDownstairs client 1 not running, waiting for it
7102024-11-01T16:53:54.714ZDownstairs client 1 not running, waiting for it
7112024-11-01T16:53:59.774ZDownstairs client 1 not running, waiting for it
7122024-11-01T16:54:04.835ZDownstairs client 1 not running, waiting for it
7132024-11-01T16:54:09.895ZDownstairs client 1 not running, waiting for it
7142024-11-01T16:54:14.956ZDownstairs client 1 not running, waiting for it
7152024-11-01T16:54:20.018ZDownstairs client 1 not running, waiting for it
7162024-11-01T16:54:25.078ZDownstairs client 1 not running, waiting for it
7172024-11-01T16:54:30.139ZDownstairs client 1 not running, waiting for it
7182024-11-01T16:54:35.200ZDownstairs client 1 not running, waiting for it
7192024-11-01T16:54:40.262ZDownstairs client 1 not running, waiting for it
7202024-11-01T16:54:45.324ZDownstairs client 1 not running, waiting for it
7212024-11-01T16:54:50.383ZDownstairs client 1 not running, waiting for it
7222024-11-01T16:54:55.444ZDownstairs client 1 not running, waiting for it
7232024-11-01T16:55:00.505ZDownstairs client 1 not running, waiting for it
7242024-11-01T16:55:05.566ZDownstairs client 1 not running, waiting for it
7252024-11-01T16:55:10.631ZDownstairs client 1 not running, waiting for it
7262024-11-01T16:55:15.690ZDownstairs client 1 not running, waiting for it
7272024-11-01T16:55:20.749ZDownstairs client 1 not running, waiting for it
7282024-11-01T16:55:25.810ZDownstairs client 1 not running, waiting for it
7292024-11-01T16:55:30.870ZDownstairs client 1 not running, waiting for it
7302024-11-01T16:55:35.929ZDownstairs client 1 not running, waiting for it
7312024-11-01T16:55:40.989ZDownstairs client 1 not running, waiting for it
7322024-11-01T16:55:46.049ZDownstairs client 1 not running, waiting for it
7332024-11-01T16:55:51.110ZDownstairs client 1 not running, waiting for it
7342024-11-01T16:55:56.171ZDownstairs client 1 not running, waiting for it
7352024-11-01T16:56:01.231ZDownstairs client 1 not running, waiting for it
7362024-11-01T16:56:06.290ZDownstairs client 1 not running, waiting for it
7372024-11-01T16:56:11.350ZDownstairs client 1 not running, waiting for it
7382024-11-01T16:56:16.411ZDownstairs client 1 not running, waiting for it
7392024-11-01T16:56:21.472ZDownstairs client 1 not running, waiting for it
7402024-11-01T16:56:26.532ZDownstairs client 1 not running, waiting for it
7412024-11-01T16:56:31.593ZDownstairs client 1 not running, waiting for it
7422024-11-01T16:56:36.654ZDownstairs client 1 not running, waiting for it
7432024-11-01T16:56:41.715ZDownstairs client 1 not running, waiting for it
7442024-11-01T16:56:46.776ZDownstairs client 1 not running, waiting for it
7452024-11-01T16:56:51.837ZDownstairs client 1 not running, waiting for it
7462024-11-01T16:56:56.898ZDownstairs client 1 not running, waiting for it
7472024-11-01T16:57:01.959ZDownstairs client 1 not running, waiting for it
7482024-11-01T16:57:07.019ZDownstairs client 1 not running, waiting for it
7492024-11-01T16:57:12.079ZDownstairs client 1 not running, waiting for it
7502024-11-01T16:57:17.141ZDownstairs client 1 not running, waiting for it
7512024-11-01T16:57:22.205ZDownstairs client 1 not running, waiting for it
7522024-11-01T16:57:27.263ZDownstairs client 1 not running, waiting for it
7532024-11-01T16:57:32.324ZDownstairs client 1 not running, waiting for it
7542024-11-01T16:57:37.384ZDownstairs client 1 not running, waiting for it
7552024-11-01T16:57:42.444ZDownstairs client 1 not running, waiting for it
7562024-11-01T16:57:47.504ZDownstairs client 1 not running, waiting for it
7572024-11-01T16:57:52.565ZDownstairs client 1 not running, waiting for it
7582024-11-01T16:57:57.625ZDownstairs client 1 not running, waiting for it
7592024-11-01T16:58:02.685ZDownstairs client 1 not running, waiting for it
7602024-11-01T16:58:07.746ZDownstairs client 1 not running, waiting for it
7612024-11-01T16:58:12.806ZDownstairs client 1 not running, waiting for it
7622024-11-01T16:58:17.867ZDownstairs client 1 not running, waiting for it
7632024-11-01T16:58:22.927ZDownstairs client 1 not running, waiting for it
7642024-11-01T16:58:27.989ZDownstairs client 1 not running, waiting for it
7652024-11-01T16:58:33.050ZDownstairs client 1 not running, waiting for it
7662024-11-01T16:58:38.111ZDownstairs client 1 not running, waiting for it
7672024-11-01T16:58:43.172ZDownstairs client 1 not running, waiting for it
7682024-11-01T16:58:48.233ZDownstairs client 1 not running, waiting for it
7692024-11-01T16:58:53.293ZDownstairs client 1 not running, waiting for it
7702024-11-01T16:58:58.354ZDownstairs client 1 not running, waiting for it
7712024-11-01T16:59:03.414ZDownstairs client 1 not running, waiting for it
7722024-11-01T16:59:08.475ZDownstairs client 1 not running, waiting for it
7732024-11-01T16:59:13.535ZDownstairs client 1 not running, waiting for it
7742024-11-01T16:59:18.596ZDownstairs client 1 not running, waiting for it
7752024-11-01T16:59:23.656ZDownstairs client 1 not running, waiting for it
7762024-11-01T16:59:28.717ZDownstairs client 1 not running, waiting for it
7772024-11-01T16:59:33.777ZDownstairs client 1 not running, waiting for it
7782024-11-01T16:59:38.837ZDownstairs client 1 not running, waiting for it
7792024-11-01T16:59:43.898ZDownstairs client 1 not running, waiting for it
7802024-11-01T16:59:48.958ZDownstairs client 1 not running, waiting for it
7812024-11-01T16:59:54.018ZDownstairs client 1 not running, waiting for it
7822024-11-01T16:59:59.079ZDownstairs client 1 not running, waiting for it
7832024-11-01T17:00:04.139ZDownstairs client 1 not running, waiting for it
7842024-11-01T17:00:09.199ZDownstairs client 1 not running, waiting for it
7852024-11-01T17:00:14.260ZDownstairs client 1 not running, waiting for it
7862024-11-01T17:00:19.321ZDownstairs client 1 not running, waiting for it
7872024-11-01T17:00:24.381ZDownstairs client 1 not running, waiting for it
7882024-11-01T17:00:29.441ZDownstairs client 1 not running, waiting for it
7892024-11-01T17:00:34.501ZDownstairs client 1 not running, waiting for it
7902024-11-01T17:00:39.562ZDownstairs client 1 not running, waiting for it
7912024-11-01T17:00:44.622ZDownstairs client 1 not running, waiting for it
7922024-11-01T17:00:49.682ZDownstairs client 1 not running, waiting for it
7932024-11-01T17:00:54.742ZDownstairs client 1 not running, waiting for it
7942024-11-01T17:00:59.803ZDownstairs client 1 not running, waiting for it
7952024-11-01T17:01:04.864ZDownstairs client 1 not running, waiting for it
7962024-11-01T17:01:09.925ZDownstairs client 1 not running, waiting for it
7972024-11-01T17:01:14.991ZDownstairs client 1 not running, waiting for it
7982024-11-01T17:01:20.046ZDownstairs client 1 not running, waiting for it
7992024-11-01T17:01:25.106ZDownstairs client 1 not running, waiting for it
8002024-11-01T17:01:30.167ZDownstairs client 1 not running, waiting for it
8012024-11-01T17:01:35.228ZDownstairs client 1 not running, waiting for it
8022024-11-01T17:01:40.289ZDownstairs client 1 not running, waiting for it
8032024-11-01T17:01:45.349ZDownstairs client 1 not running, waiting for it
8042024-11-01T17:01:50.410ZDownstairs client 1 not running, waiting for it
8052024-11-01T17:01:55.470ZDownstairs client 1 not running, waiting for it
8062024-11-01T17:02:00.531ZDownstairs client 1 not running, waiting for it
8072024-11-01T17:02:05.590ZDownstairs client 1 not running, waiting for it
8082024-11-01T17:02:10.650ZDownstairs client 1 not running, waiting for it
8092024-11-01T17:02:15.710ZDownstairs client 1 not running, waiting for it
8102024-11-01T17:02:20.771ZDownstairs client 1 not running, waiting for it
8112024-11-01T17:02:25.832ZDownstairs client 1 not running, waiting for it
8122024-11-01T17:02:30.890ZDownstairs client 1 not running, waiting for it
8132024-11-01T17:02:35.951ZDownstairs client 1 not running, waiting for it
8142024-11-01T17:02:41.011ZDownstairs client 1 not running, waiting for it
8152024-11-01T17:02:46.072ZDownstairs client 1 not running, waiting for it
8162024-11-01T17:02:51.133ZDownstairs client 1 not running, waiting for it
8172024-11-01T17:02:56.193ZDownstairs client 1 not running, waiting for it
8182024-11-01T17:03:01.254ZDownstairs client 1 not running, waiting for it
8192024-11-01T17:03:06.315ZDownstairs client 1 not running, waiting for it
8202024-11-01T17:03:11.375ZDownstairs client 1 not running, waiting for it
8212024-11-01T17:03:16.436ZDownstairs client 1 not running, waiting for it
8222024-11-01T17:03:21.497ZDownstairs client 1 not running, waiting for it
8232024-11-01T17:03:26.558ZDownstairs client 1 not running, waiting for it
8242024-11-01T17:03:31.618ZDownstairs client 1 not running, waiting for it
8252024-11-01T17:03:36.679ZDownstairs client 1 not running, waiting for it
8262024-11-01T17:03:41.740ZDownstairs client 1 not running, waiting for it
8272024-11-01T17:03:46.801ZDownstairs client 1 not running, waiting for it
8282024-11-01T17:03:51.862ZDownstairs client 1 not running, waiting for it
8292024-11-01T17:03:56.922ZDownstairs client 1 not running, waiting for it
8302024-11-01T17:04:01.982ZDownstairs client 1 not running, waiting for it
8312024-11-01T17:04:07.042ZDownstairs client 1 not running, waiting for it
8322024-11-01T17:04:12.103ZDownstairs client 1 not running, waiting for it
8332024-11-01T17:04:17.165ZDownstairs client 1 not running, waiting for it
8342024-11-01T17:04:22.225ZDownstairs client 1 not running, waiting for it
8352024-11-01T17:04:27.288ZDownstairs client 1 not running, waiting for it
8362024-11-01T17:04:32.349ZDownstairs client 1 not running, waiting for it
8372024-11-01T17:04:37.409ZDownstairs client 1 not running, waiting for it
8382024-11-01T17:04:42.469ZDownstairs client 1 not running, waiting for it
8392024-11-01T17:04:47.529ZDownstairs client 1 not running, waiting for it
8402024-11-01T17:04:52.588ZDownstairs client 1 not running, waiting for it
8412024-11-01T17:04:57.650ZDownstairs client 1 not running, waiting for it
8422024-11-01T17:05:02.708ZDownstairs client 1 not running, waiting for it
8432024-11-01T17:05:07.767ZDownstairs client 1 not running, waiting for it
8442024-11-01T17:05:12.827ZDownstairs client 1 not running, waiting for it
8452024-11-01T17:05:17.888ZDownstairs client 1 not running, waiting for it
8462024-11-01T17:05:22.948ZDownstairs client 1 not running, waiting for it
8472024-11-01T17:05:28.010ZDownstairs client 1 not running, waiting for it
8482024-11-01T17:05:33.071ZDownstairs client 1 not running, waiting for it
8492024-11-01T17:05:38.131ZDownstairs client 1 not running, waiting for it
8502024-11-01T17:05:43.193ZDownstairs client 1 not running, waiting for it
8512024-11-01T17:05:48.254ZDownstairs client 1 not running, waiting for it
8522024-11-01T17:05:53.314ZDownstairs client 1 not running, waiting for it
8532024-11-01T17:05:58.374ZDownstairs client 1 not running, waiting for it
8542024-11-01T17:06:03.434ZDownstairs client 1 not running, waiting for it
8552024-11-01T17:06:08.494ZDownstairs client 1 not running, waiting for it
8562024-11-01T17:06:13.555ZDownstairs client 1 not running, waiting for it
8572024-11-01T17:06:18.614ZDownstairs client 1 not running, waiting for it
8582024-11-01T17:06:23.675ZDownstairs client 1 not running, waiting for it
8592024-11-01T17:06:28.734ZDownstairs client 1 not running, waiting for it
8602024-11-01T17:06:33.795ZDownstairs client 1 not running, waiting for it
8612024-11-01T17:06:38.856ZDownstairs client 1 not running, waiting for it
8622024-11-01T17:06:43.916ZDownstairs client 1 not running, waiting for it
8632024-11-01T17:06:48.976ZDownstairs client 1 not running, waiting for it
8642024-11-01T17:06:54.035ZDownstairs client 1 not running, waiting for it
8652024-11-01T17:06:59.096ZDownstairs client 1 not running, waiting for it
8662024-11-01T17:07:04.156ZDownstairs client 1 not running, waiting for it
8672024-11-01T17:07:09.216ZDownstairs client 1 not running, waiting for it
8682024-11-01T17:07:14.278ZDownstairs client 1 not running, waiting for it
8692024-11-01T17:07:19.336ZDownstairs client 1 not running, waiting for it
8702024-11-01T17:07:24.396ZDownstairs client 1 not running, waiting for it
8712024-11-01T17:07:29.456ZDownstairs client 1 not running, waiting for it
8722024-11-01T17:07:34.517ZDownstairs client 1 not running, waiting for it
8732024-11-01T17:07:39.578ZDownstairs client 1 not running, waiting for it
8742024-11-01T17:07:44.638ZDownstairs client 1 not running, waiting for it
8752024-11-01T17:07:49.698ZDownstairs client 1 not running, waiting for it
8762024-11-01T17:07:54.759ZDownstairs client 1 not running, waiting for it
8772024-11-01T17:07:59.819ZDownstairs client 1 not running, waiting for it
8782024-11-01T17:08:04.880ZDownstairs client 1 not running, waiting for it
8792024-11-01T17:08:09.940ZDownstairs client 1 not running, waiting for it
8802024-11-01T17:08:15.001ZDownstairs client 1 not running, waiting for it
8812024-11-01T17:08:20.062ZDownstairs client 1 not running, waiting for it
8822024-11-01T17:08:25.123ZDownstairs client 1 not running, waiting for it
8832024-11-01T17:08:30.184ZDownstairs client 1 not running, waiting for it
8842024-11-01T17:08:35.244ZDownstairs client 1 not running, waiting for it
8852024-11-01T17:08:40.305ZDownstairs client 1 not running, waiting for it
8862024-11-01T17:08:45.366ZDownstairs client 1 not running, waiting for it
8872024-11-01T17:08:50.427ZDownstairs client 1 not running, waiting for it
8882024-11-01T17:08:55.489ZDownstairs client 1 not running, waiting for it
8892024-11-01T17:09:00.550ZDownstairs client 1 not running, waiting for it
8902024-11-01T17:09:05.610ZDownstairs client 1 not running, waiting for it
8912024-11-01T17:09:10.672ZDownstairs client 1 not running, waiting for it
8922024-11-01T17:09:15.732ZDownstairs client 1 not running, waiting for it
8932024-11-01T17:09:20.792ZDownstairs client 1 not running, waiting for it
8942024-11-01T17:09:25.852ZDownstairs client 1 not running, waiting for it
8952024-11-01T17:09:30.913ZDownstairs client 1 not running, waiting for it
8962024-11-01T17:09:35.973ZDownstairs client 1 not running, waiting for it
8972024-11-01T17:09:41.034ZDownstairs client 1 not running, waiting for it
8982024-11-01T17:09:46.095ZDownstairs client 1 not running, waiting for it
8992024-11-01T17:09:51.156ZDownstairs client 1 not running, waiting for it
9002024-11-01T17:09:56.217ZDownstairs client 1 not running, waiting for it
9012024-11-01T17:10:01.278ZDownstairs client 1 not running, waiting for it
9022024-11-01T17:10:06.340ZDownstairs client 1 not running, waiting for it
9032024-11-01T17:10:11.401ZDownstairs client 1 not running, waiting for it
9042024-11-01T17:10:16.462ZDownstairs client 1 not running, waiting for it
9052024-11-01T17:10:21.523ZDownstairs client 1 not running, waiting for it
9062024-11-01T17:10:26.584ZDownstairs client 1 not running, waiting for it
9072024-11-01T17:10:31.645ZDownstairs client 1 not running, waiting for it
9082024-11-01T17:10:36.706ZDownstairs client 1 not running, waiting for it
9092024-11-01T17:10:41.766ZDownstairs client 1 not running, waiting for it
9102024-11-01T17:10:46.828ZDownstairs client 1 not running, waiting for it
9112024-11-01T17:10:51.887ZDownstairs client 1 not running, waiting for it
9122024-11-01T17:10:56.948ZDownstairs client 1 not running, waiting for it
9132024-11-01T17:11:02.008ZDownstairs client 1 not running, waiting for it
9142024-11-01T17:11:07.068ZDownstairs client 1 not running, waiting for it
9152024-11-01T17:11:12.129ZDownstairs client 1 not running, waiting for it
9162024-11-01T17:11:17.188ZDownstairs client 1 not running, waiting for it
9172024-11-01T17:11:22.248ZDownstairs client 1 not running, waiting for it
9182024-11-01T17:11:27.309ZDownstairs client 1 not running, waiting for it
9192024-11-01T17:11:32.369ZDownstairs client 1 not running, waiting for it
9202024-11-01T17:11:37.430ZDownstairs client 1 not running, waiting for it
9212024-11-01T17:11:42.491ZDownstairs client 1 not running, waiting for it
9222024-11-01T17:11:47.550ZDownstairs client 1 not running, waiting for it
9232024-11-01T17:11:52.611ZDownstairs client 1 not running, waiting for it
9242024-11-01T17:11:57.671ZDownstairs client 1 not running, waiting for it
9252024-11-01T17:12:02.730ZDownstairs client 1 not running, waiting for it
9262024-11-01T17:12:07.792ZDownstairs client 1 not running, waiting for it
9272024-11-01T17:12:12.852ZDownstairs client 1 not running, waiting for it
9282024-11-01T17:12:17.913ZDownstairs client 1 not running, waiting for it
9292024-11-01T17:12:22.973ZDownstairs client 1 not running, waiting for it
9302024-11-01T17:12:28.033ZDownstairs client 1 not running, waiting for it
9312024-11-01T17:12:33.095ZDownstairs client 1 not running, waiting for it
9322024-11-01T17:12:38.154ZDownstairs client 1 not running, waiting for it
9332024-11-01T17:12:43.214ZDownstairs client 1 not running, waiting for it
9342024-11-01T17:12:48.274ZDownstairs client 1 not running, waiting for it
9352024-11-01T17:12:53.336ZDownstairs client 1 not running, waiting for it
9362024-11-01T17:12:58.397ZDownstairs client 1 not running, waiting for it
9372024-11-01T17:13:03.458ZDownstairs client 1 not running, waiting for it
9382024-11-01T17:13:08.520ZDownstairs client 1 not running, waiting for it
9392024-11-01T17:13:13.581ZDownstairs client 1 not running, waiting for it
9402024-11-01T17:13:18.640ZDownstairs client 1 not running, waiting for it
9412024-11-01T17:13:23.702ZDownstairs client 1 not running, waiting for it
9422024-11-01T17:13:28.763ZDownstairs client 1 not running, waiting for it
9432024-11-01T17:13:33.824ZDownstairs client 1 not running, waiting for it
9442024-11-01T17:13:38.884ZDownstairs client 1 not running, waiting for it
9452024-11-01T17:13:43.943ZDownstairs client 1 not running, waiting for it
9462024-11-01T17:13:49.003ZDownstairs client 1 not running, waiting for it
9472024-11-01T17:13:54.064ZDownstairs client 1 not running, waiting for it
9482024-11-01T17:13:59.124ZDownstairs client 1 not running, waiting for it
9492024-11-01T17:14:04.185ZDownstairs client 1 not running, waiting for it
9502024-11-01T17:14:09.246ZDownstairs client 1 not running, waiting for it
9512024-11-01T17:14:14.307ZDownstairs client 1 not running, waiting for it
9522024-11-01T17:14:19.367ZDownstairs client 1 not running, waiting for it
9532024-11-01T17:14:24.428ZDownstairs client 1 not running, waiting for it
9542024-11-01T17:14:29.490ZDownstairs client 1 not running, waiting for it
9552024-11-01T17:14:34.550ZDownstairs client 1 not running, waiting for it
9562024-11-01T17:14:39.611ZDownstairs client 1 not running, waiting for it
9572024-11-01T17:14:44.673ZDownstairs client 1 not running, waiting for it
9582024-11-01T17:14:49.734ZDownstairs client 1 not running, waiting for it
9592024-11-01T17:14:54.796ZDownstairs client 1 not running, waiting for it
9602024-11-01T17:14:59.857ZDownstairs client 1 not running, waiting for it
9612024-11-01T17:15:04.917ZDownstairs client 1 not running, waiting for it
9622024-11-01T17:15:09.978ZDownstairs client 1 not running, waiting for it
9632024-11-01T17:15:15.039ZDownstairs client 1 not running, waiting for it
9642024-11-01T17:15:20.100ZDownstairs client 1 not running, waiting for it
9652024-11-01T17:15:25.161ZDownstairs client 1 not running, waiting for it
9662024-11-01T17:15:30.221ZDownstairs client 1 not running, waiting for it
9672024-11-01T17:15:35.281ZDownstairs client 1 not running, waiting for it
9682024-11-01T17:15:40.342ZDownstairs client 1 not running, waiting for it
9692024-11-01T17:15:45.403ZDownstairs client 1 not running, waiting for it
9702024-11-01T17:15:50.464ZDownstairs client 1 not running, waiting for it
9712024-11-01T17:15:55.525ZDownstairs client 1 not running, waiting for it
9722024-11-01T17:16:00.586ZDownstairs client 1 not running, waiting for it
9732024-11-01T17:16:05.647ZDownstairs client 1 not running, waiting for it
9742024-11-01T17:16:10.708ZDownstairs client 1 not running, waiting for it
9752024-11-01T17:16:15.768ZDownstairs client 1 not running, waiting for it
9762024-11-01T17:16:20.828ZDownstairs client 1 not running, waiting for it
9772024-11-01T17:16:25.888ZDownstairs client 1 not running, waiting for it
9782024-11-01T17:16:30.949ZDownstairs client 1 not running, waiting for it
9792024-11-01T17:16:36.010ZDownstairs client 1 not running, waiting for it
9802024-11-01T17:16:41.070ZDownstairs client 1 not running, waiting for it
9812024-11-01T17:16:46.130ZDownstairs client 1 not running, waiting for it
9822024-11-01T17:16:51.191ZDownstairs client 1 not running, waiting for it
9832024-11-01T17:16:56.252ZDownstairs client 1 not running, waiting for it
9842024-11-01T17:17:01.313ZDownstairs client 1 not running, waiting for it
9852024-11-01T17:17:06.375ZDownstairs client 1 not running, waiting for it
9862024-11-01T17:17:11.436ZDownstairs client 1 not running, waiting for it
9872024-11-01T17:17:16.496ZDownstairs client 1 not running, waiting for it
9882024-11-01T17:17:21.556ZDownstairs client 1 not running, waiting for it
9892024-11-01T17:17:26.616ZDownstairs client 1 not running, waiting for it
9902024-11-01T17:17:31.678ZDownstairs client 1 not running, waiting for it
9912024-11-01T17:17:36.739ZDownstairs client 1 not running, waiting for it
9922024-11-01T17:17:41.798ZDownstairs client 1 not running, waiting for it
9932024-11-01T17:17:46.860ZDownstairs client 1 not running, waiting for it
9942024-11-01T17:17:51.921ZDownstairs client 1 not running, waiting for it
9952024-11-01T17:17:56.981ZDownstairs client 1 not running, waiting for it
9962024-11-01T17:18:02.043ZDownstairs client 1 not running, waiting for it
9972024-11-01T17:18:07.103ZDownstairs client 1 not running, waiting for it
9982024-11-01T17:18:12.163ZDownstairs client 1 not running, waiting for it
9992024-11-01T17:18:17.225ZDownstairs client 1 not running, waiting for it
10002024-11-01T17:18:22.286ZDownstairs client 1 not running, waiting for it
10012024-11-01T17:18:27.346ZDownstairs client 1 not running, waiting for it
10022024-11-01T17:18:32.407ZDownstairs client 1 not running, waiting for it
10032024-11-01T17:18:37.467ZDownstairs client 1 not running, waiting for it
10042024-11-01T17:18:42.528ZDownstairs client 1 not running, waiting for it
10052024-11-01T17:18:47.588ZDownstairs client 1 not running, waiting for it
10062024-11-01T17:18:52.648ZDownstairs client 1 not running, waiting for it
10072024-11-01T17:18:57.710ZDownstairs client 1 not running, waiting for it
10082024-11-01T17:19:02.770ZDownstairs client 1 not running, waiting for it
10092024-11-01T17:19:07.831ZDownstairs client 1 not running, waiting for it
10102024-11-01T17:19:12.894ZDownstairs client 1 not running, waiting for it
10112024-11-01T17:19:17.952ZDownstairs client 1 not running, waiting for it
10122024-11-01T17:19:23.012ZDownstairs client 1 not running, waiting for it
10132024-11-01T17:19:28.078ZDownstairs client 1 not running, waiting for it
10142024-11-01T17:19:33.133ZDownstairs client 1 not running, waiting for it
10152024-11-01T17:19:38.194ZDownstairs client 1 not running, waiting for it
10162024-11-01T17:19:43.255ZDownstairs client 1 not running, waiting for it
10172024-11-01T17:19:48.315ZDownstairs client 1 not running, waiting for it
10182024-11-01T17:19:53.376ZDownstairs client 1 not running, waiting for it
10192024-11-01T17:19:58.437ZDownstairs client 1 not running, waiting for it
10202024-11-01T17:20:03.497ZDownstairs client 1 not running, waiting for it
10212024-11-01T17:20:08.556ZDownstairs client 1 not running, waiting for it
10222024-11-01T17:20:13.617ZDownstairs client 1 not running, waiting for it
10232024-11-01T17:20:18.677ZDownstairs client 1 not running, waiting for it
10242024-11-01T17:20:23.737ZDownstairs client 1 not running, waiting for it
10252024-11-01T17:20:28.798ZDownstairs client 1 not running, waiting for it
10262024-11-01T17:20:33.859ZDownstairs client 1 not running, waiting for it
10272024-11-01T17:20:38.919ZDownstairs client 1 not running, waiting for it
10282024-11-01T17:20:43.980ZDownstairs client 1 not running, waiting for it
10292024-11-01T17:20:49.040ZDownstairs client 1 not running, waiting for it
10302024-11-01T17:20:54.100ZDownstairs client 1 not running, waiting for it
10312024-11-01T17:20:59.159ZDownstairs client 1 not running, waiting for it
10322024-11-01T17:21:04.219ZDownstairs client 1 not running, waiting for it
10332024-11-01T17:21:09.279ZDownstairs client 1 not running, waiting for it
10342024-11-01T17:21:14.339ZDownstairs client 1 not running, waiting for it
10352024-11-01T17:21:19.398ZDownstairs client 1 not running, waiting for it
10362024-11-01T17:21:24.460ZDownstairs client 1 not running, waiting for it
10372024-11-01T17:21:29.520ZDownstairs client 1 not running, waiting for it
10382024-11-01T17:21:34.580ZDownstairs client 1 not running, waiting for it
10392024-11-01T17:21:39.640ZDownstairs client 1 not running, waiting for it
10402024-11-01T17:21:44.699ZDownstairs client 1 not running, waiting for it
10412024-11-01T17:21:49.760ZDownstairs client 1 not running, waiting for it
10422024-11-01T17:21:54.822ZDownstairs client 1 not running, waiting for it
10432024-11-01T17:21:59.882ZDownstairs client 1 not running, waiting for it
10442024-11-01T17:22:04.942ZDownstairs client 1 not running, waiting for it
10452024-11-01T17:22:10.002ZDownstairs client 1 not running, waiting for it
10462024-11-01T17:22:15.062ZDownstairs client 1 not running, waiting for it
10472024-11-01T17:22:20.122ZDownstairs client 1 not running, waiting for it
10482024-11-01T17:22:25.183ZDownstairs client 1 not running, waiting for it
10492024-11-01T17:22:30.243ZDownstairs client 1 not running, waiting for it
10502024-11-01T17:22:35.304ZDownstairs client 1 not running, waiting for it
10512024-11-01T17:22:40.364ZDownstairs client 1 not running, waiting for it
10522024-11-01T17:22:45.426ZDownstairs client 1 not running, waiting for it
10532024-11-01T17:22:50.487ZDownstairs client 1 not running, waiting for it
10542024-11-01T17:22:55.547ZDownstairs client 1 not running, waiting for it
10552024-11-01T17:23:00.608ZDownstairs client 1 not running, waiting for it
10562024-11-01T17:23:05.669ZDownstairs client 1 not running, waiting for it
10572024-11-01T17:23:10.730ZDownstairs client 1 not running, waiting for it
10582024-11-01T17:23:15.790ZDownstairs client 1 not running, waiting for it
10592024-11-01T17:23:20.851ZDownstairs client 1 not running, waiting for it
10602024-11-01T17:23:25.911ZDownstairs client 1 not running, waiting for it
10612024-11-01T17:23:30.972ZDownstairs client 1 not running, waiting for it
10622024-11-01T17:23:36.032ZDownstairs client 1 not running, waiting for it
10632024-11-01T17:23:41.093ZDownstairs client 1 not running, waiting for it
10642024-11-01T17:23:46.153ZDownstairs client 1 not running, waiting for it
10652024-11-01T17:23:51.214ZDownstairs client 1 not running, waiting for it
10662024-11-01T17:23:56.275ZDownstairs client 1 not running, waiting for it
10672024-11-01T17:24:01.339ZDownstairs client 1 not running, waiting for it
10682024-11-01T17:24:06.396ZDownstairs client 1 not running, waiting for it
10692024-11-01T17:24:11.457ZDownstairs client 1 not running, waiting for it
10702024-11-01T17:24:16.517ZDownstairs client 1 not running, waiting for it
10712024-11-01T17:24:21.577ZDownstairs client 1 not running, waiting for it
10722024-11-01T17:24:26.638ZDownstairs client 1 not running, waiting for it
10732024-11-01T17:24:31.699ZDownstairs client 1 not running, waiting for it
10742024-11-01T17:24:36.760ZDownstairs client 1 not running, waiting for it
10752024-11-01T17:24:41.820ZDownstairs client 1 not running, waiting for it
10762024-11-01T17:24:46.881ZDownstairs client 1 not running, waiting for it
10772024-11-01T17:24:51.941ZDownstairs client 1 not running, waiting for it
10782024-11-01T17:24:57.002ZDownstairs client 1 not running, waiting for it
10792024-11-01T17:25:02.062ZDownstairs client 1 not running, waiting for it
10802024-11-01T17:25:07.123ZDownstairs client 1 not running, waiting for it
10812024-11-01T17:25:12.185ZDownstairs client 1 not running, waiting for it
10822024-11-01T17:25:17.244ZDownstairs client 1 not running, waiting for it
10832024-11-01T17:25:22.306ZDownstairs client 1 not running, waiting for it
10842024-11-01T17:25:27.365ZDownstairs client 1 not running, waiting for it
10852024-11-01T17:25:32.426ZDownstairs client 1 not running, waiting for it
10862024-11-01T17:25:37.486ZDownstairs client 1 not running, waiting for it
10872024-11-01T17:25:42.547ZDownstairs client 1 not running, waiting for it
10882024-11-01T17:25:47.609ZDownstairs client 1 not running, waiting for it
10892024-11-01T17:25:52.670ZDownstairs client 1 not running, waiting for it
10902024-11-01T17:25:57.731ZDownstairs client 1 not running, waiting for it
10912024-11-01T17:26:02.791ZDownstairs client 1 not running, waiting for it
10922024-11-01T17:26:07.851ZDownstairs client 1 not running, waiting for it
10932024-11-01T17:26:12.911ZDownstairs client 1 not running, waiting for it
10942024-11-01T17:26:17.971ZDownstairs client 1 not running, waiting for it
10952024-11-01T17:26:23.032ZDownstairs client 1 not running, waiting for it
10962024-11-01T17:26:28.093ZDownstairs client 1 not running, waiting for it
10972024-11-01T17:26:33.153ZDownstairs client 1 not running, waiting for it
10982024-11-01T17:26:38.213ZDownstairs client 1 not running, waiting for it
10992024-11-01T17:26:43.274ZDownstairs client 1 not running, waiting for it
11002024-11-01T17:26:48.335ZDownstairs client 1 not running, waiting for it
11012024-11-01T17:26:53.396ZDownstairs client 1 not running, waiting for it
11022024-11-01T17:26:58.456ZDownstairs client 1 not running, waiting for it
11032024-11-01T17:27:03.518ZDownstairs client 1 not running, waiting for it
11042024-11-01T17:27:08.578ZDownstairs client 1 not running, waiting for it
11052024-11-01T17:27:13.638ZDownstairs client 1 not running, waiting for it
11062024-11-01T17:27:18.698ZDownstairs client 1 not running, waiting for it
11072024-11-01T17:27:23.759ZDownstairs client 1 not running, waiting for it
11082024-11-01T17:27:28.818ZDownstairs client 1 not running, waiting for it
11092024-11-01T17:27:33.879ZDownstairs client 1 not running, waiting for it
11102024-11-01T17:27:38.940ZDownstairs client 1 not running, waiting for it
11112024-11-01T17:27:44.001ZDownstairs client 1 not running, waiting for it
11122024-11-01T17:27:49.061ZDownstairs client 1 not running, waiting for it
11132024-11-01T17:27:54.122ZDownstairs client 1 not running, waiting for it
11142024-11-01T17:27:59.181ZDownstairs client 1 not running, waiting for it
11152024-11-01T17:28:04.241ZDownstairs client 1 not running, waiting for it
11162024-11-01T17:28:09.302ZDownstairs client 1 not running, waiting for it
11172024-11-01T17:28:14.362ZDownstairs client 1 not running, waiting for it
11182024-11-01T17:28:19.422ZDownstairs client 1 not running, waiting for it
11192024-11-01T17:28:24.484ZDownstairs client 1 not running, waiting for it
11202024-11-01T17:28:29.545ZDownstairs client 1 not running, waiting for it
11212024-11-01T17:28:34.606ZDownstairs client 1 not running, waiting for it
11222024-11-01T17:28:39.667ZDownstairs client 1 not running, waiting for it
11232024-11-01T17:28:44.726ZDownstairs client 1 not running, waiting for it
11242024-11-01T17:28:49.787ZDownstairs client 1 not running, waiting for it
11252024-11-01T17:28:54.847ZDownstairs client 1 not running, waiting for it
11262024-11-01T17:28:59.907ZDownstairs client 1 not running, waiting for it
11272024-11-01T17:29:04.968ZDownstairs client 1 not running, waiting for it
11282024-11-01T17:29:10.028ZDownstairs client 1 not running, waiting for it
11292024-11-01T17:29:15.088ZDownstairs client 1 not running, waiting for it
11302024-11-01T17:29:20.147ZDownstairs client 1 not running, waiting for it
11312024-11-01T17:29:25.208ZDownstairs client 1 not running, waiting for it
11322024-11-01T17:29:30.269ZDownstairs client 1 not running, waiting for it
11332024-11-01T17:29:35.329ZDownstairs client 1 not running, waiting for it
11342024-11-01T17:29:40.390ZDownstairs client 1 not running, waiting for it
11352024-11-01T17:29:45.451ZDownstairs client 1 not running, waiting for it
11362024-11-01T17:29:50.513ZDownstairs client 1 not running, waiting for it
11372024-11-01T17:29:55.574ZDownstairs client 1 not running, waiting for it
11382024-11-01T17:30:00.635ZDownstairs client 1 not running, waiting for it
11392024-11-01T17:30:05.696ZDownstairs client 1 not running, waiting for it
11402024-11-01T17:30:10.757ZDownstairs client 1 not running, waiting for it
11412024-11-01T17:30:15.817ZDownstairs client 1 not running, waiting for it
11422024-11-01T17:30:20.879ZDownstairs client 1 not running, waiting for it
11432024-11-01T17:30:25.940ZDownstairs client 1 not running, waiting for it
11442024-11-01T17:30:31.002ZDownstairs client 1 not running, waiting for it
11452024-11-01T17:30:36.062ZDownstairs client 1 not running, waiting for it
11462024-11-01T17:30:41.187ZDownstairs client 1 not running, waiting for it
11472024-11-01T17:30:46.191ZDownstairs client 1 not running, waiting for it
11482024-11-01T17:30:51.252ZDownstairs client 1 not running, waiting for it
11492024-11-01T17:30:56.306ZDownstairs client 1 not running, waiting for it
11502024-11-01T17:31:01.368ZDownstairs client 1 not running, waiting for it
11512024-11-01T17:31:06.429ZDownstairs client 1 not running, waiting for it
11522024-11-01T17:31:11.491ZDownstairs client 1 not running, waiting for it
11532024-11-01T17:31:16.552ZDownstairs client 1 not running, waiting for it
11542024-11-01T17:31:21.613ZDownstairs client 1 not running, waiting for it
11552024-11-01T17:31:26.674ZDownstairs client 1 not running, waiting for it
11562024-11-01T17:31:31.734ZDownstairs client 1 not running, waiting for it
11572024-11-01T17:31:36.796ZDownstairs client 1 not running, waiting for it
11582024-11-01T17:31:41.856ZDownstairs client 1 not running, waiting for it
11592024-11-01T17:31:46.917ZDownstairs client 1 not running, waiting for it
11602024-11-01T17:31:51.978ZDownstairs client 1 not running, waiting for it
11612024-11-01T17:31:57.039ZDownstairs client 1 not running, waiting for it
11622024-11-01T17:32:02.100ZDownstairs client 1 not running, waiting for it
11632024-11-01T17:32:07.161ZDownstairs client 1 not running, waiting for it
11642024-11-01T17:32:12.220ZDownstairs client 1 not running, waiting for it
11652024-11-01T17:32:17.282ZDownstairs client 1 not running, waiting for it
11662024-11-01T17:32:22.342ZDownstairs client 1 not running, waiting for it
11672024-11-01T17:32:27.403ZDownstairs client 1 not running, waiting for it
11682024-11-01T17:32:32.464ZDownstairs client 1 not running, waiting for it
11692024-11-01T17:32:37.526ZDownstairs client 1 not running, waiting for it
11702024-11-01T17:32:42.586ZDownstairs client 1 not running, waiting for it
11712024-11-01T17:32:47.647ZDownstairs client 1 not running, waiting for it
11722024-11-01T17:32:52.708ZDownstairs client 1 not running, waiting for it
11732024-11-01T17:32:57.769ZDownstairs client 1 not running, waiting for it
11742024-11-01T17:33:02.829ZDownstairs client 1 not running, waiting for it
11752024-11-01T17:33:07.891ZDownstairs client 1 not running, waiting for it
11762024-11-01T17:33:12.951ZDownstairs client 1 not running, waiting for it
11772024-11-01T17:33:18.012ZDownstairs client 1 not running, waiting for it
11782024-11-01T17:33:23.074ZDownstairs client 1 not running, waiting for it
11792024-11-01T17:33:28.135ZDownstairs client 1 not running, waiting for it
11802024-11-01T17:33:33.197ZDownstairs client 1 not running, waiting for it
11812024-11-01T17:33:38.256ZDownstairs client 1 not running, waiting for it
11822024-11-01T17:33:43.320ZDownstairs client 1 not running, waiting for it
11832024-11-01T17:33:48.379ZDownstairs client 1 not running, waiting for it
11842024-11-01T17:33:53.486ZDownstairs client 1 not running, waiting for it
11852024-11-01T17:33:58.556ZDownstairs client 1 not running, waiting for it
11862024-11-01T17:34:03.560ZDownstairs client 1 not running, waiting for it
11872024-11-01T17:34:08.620ZDownstairs client 1 not running, waiting for it
11882024-11-01T17:34:13.684ZDownstairs client 1 not running, waiting for it
11892024-11-01T17:34:18.742ZDownstairs client 1 not running, waiting for it
11902024-11-01T17:34:23.802ZDownstairs client 1 not running, waiting for it
11912024-11-01T17:34:28.863ZDownstairs client 1 not running, waiting for it
11922024-11-01T17:34:33.925ZDownstairs client 1 not running, waiting for it
11932024-11-01T17:34:38.987ZDownstairs client 1 not running, waiting for it
11942024-11-01T17:34:44.047ZDownstairs client 1 not running, waiting for it
11952024-11-01T17:34:49.108ZDownstairs client 1 not running, waiting for it
11962024-11-01T17:34:54.168ZDownstairs client 1 not running, waiting for it
11972024-11-01T17:34:59.228ZDownstairs client 1 not running, waiting for it
11982024-11-01T17:35:04.289ZDownstairs client 1 not running, waiting for it
11992024-11-01T17:35:09.351ZDownstairs client 1 not running, waiting for it
12002024-11-01T17:35:14.411ZDownstairs client 1 not running, waiting for it
12012024-11-01T17:35:19.472ZDownstairs client 1 not running, waiting for it
12022024-11-01T17:35:24.533ZDownstairs client 1 not running, waiting for it
12032024-11-01T17:35:29.593ZDownstairs client 1 not running, waiting for it
12042024-11-01T17:35:34.653ZDownstairs client 1 not running, waiting for it
12052024-11-01T17:35:39.714ZDownstairs client 1 not running, waiting for it
12062024-11-01T17:35:44.775ZDownstairs client 1 not running, waiting for it
12072024-11-01T17:35:49.835ZDownstairs client 1 not running, waiting for it
12082024-11-01T17:35:54.897ZDownstairs client 1 not running, waiting for it
12092024-11-01T17:35:59.956ZDownstairs client 1 not running, waiting for it
12102024-11-01T17:36:05.017ZDownstairs client 1 not running, waiting for it
12112024-11-01T17:36:10.078ZDownstairs client 1 not running, waiting for it
12122024-11-01T17:36:15.140ZDownstairs client 1 not running, waiting for it
12132024-11-01T17:36:20.201ZDownstairs client 1 not running, waiting for it
12142024-11-01T17:36:25.262ZDownstairs client 1 not running, waiting for it
12152024-11-01T17:36:30.323ZDownstairs client 1 not running, waiting for it
12162024-11-01T17:36:35.384ZDownstairs client 1 not running, waiting for it
12172024-11-01T17:36:40.445ZDownstairs client 1 not running, waiting for it
12182024-11-01T17:36:45.506ZDownstairs client 1 not running, waiting for it
12192024-11-01T17:36:50.567ZDownstairs client 1 not running, waiting for it
12202024-11-01T17:36:55.628ZDownstairs client 1 not running, waiting for it
12212024-11-01T17:37:00.688ZDownstairs client 1 not running, waiting for it
12222024-11-01T17:37:05.749ZDownstairs client 1 not running, waiting for it
12232024-11-01T17:37:10.810ZDownstairs client 1 not running, waiting for it
12242024-11-01T17:37:15.871ZDownstairs client 1 not running, waiting for it
12252024-11-01T17:37:20.931ZDownstairs client 1 not running, waiting for it
12262024-11-01T17:37:25.990ZDownstairs client 1 not running, waiting for it
12272024-11-01T17:37:31.052ZDownstairs client 1 not running, waiting for it
12282024-11-01T17:37:36.111ZDownstairs client 1 not running, waiting for it
12292024-11-01T17:37:41.172ZDownstairs client 1 not running, waiting for it
12302024-11-01T17:37:46.232ZDownstairs client 1 not running, waiting for it
12312024-11-01T17:37:51.294ZDownstairs client 1 not running, waiting for it
12322024-11-01T17:37:56.353ZDownstairs client 1 not running, waiting for it
12332024-11-01T17:38:01.418ZDownstairs client 1 not running, waiting for it
12342024-11-01T17:38:06.475ZDownstairs client 1 not running, waiting for it
12352024-11-01T17:38:11.534ZDownstairs client 1 not running, waiting for it
12362024-11-01T17:38:16.596ZDownstairs client 1 not running, waiting for it
12372024-11-01T17:38:21.657ZDownstairs client 1 not running, waiting for it
12382024-11-01T17:38:26.717ZDownstairs client 1 not running, waiting for it
12392024-11-01T17:38:31.777ZDownstairs client 1 not running, waiting for it
12402024-11-01T17:38:36.838ZDownstairs client 1 not running, waiting for it
12412024-11-01T17:38:41.899ZDownstairs client 1 not running, waiting for it
12422024-11-01T17:38:46.959ZDownstairs client 1 not running, waiting for it
12432024-11-01T17:38:52.021ZDownstairs client 1 not running, waiting for it
12442024-11-01T17:38:57.082ZDownstairs client 1 not running, waiting for it
12452024-11-01T17:39:02.143ZDownstairs client 1 not running, waiting for it
12462024-11-01T17:39:07.204ZDownstairs client 1 not running, waiting for it
12472024-11-01T17:39:10.704Z+ banner fail-timeout
12482024-11-01T17:39:10.706Z
12492024-11-01T17:39:10.706Z ###### ## # # ##### # # # ###### ####
12502024-11-01T17:39:10.706Z # # # # # # # ## ## # # #
12512024-11-01T17:39:10.706Z ##### # # # # ##### # # # ## # ##### # #
12522024-11-01T17:39:10.706Z # ###### # # # # # # # # #
12532024-11-01T17:39:10.706Z # # # # # # # # # # # #
12542024-11-01T17:39:10.706Z # # # # ###### # # # # ###### ####
12552024-11-01T17:39:10.708Z
12562024-11-01T17:39:10.708Z+ ps -ef
12572024-11-01T17:39:10.713Z UID PID PPID C STIME TTY TIME CMD
12582024-11-01T17:39:10.713Z root 0 0 0 16:38:31 ? 0:01 sched
12592024-11-01T17:39:10.713Z root 4 0 0 16:38:29 ? 0:00 kcfpoold
12602024-11-01T17:39:10.713Z root 5 0 0 16:38:31 ? 0:09 zpool-rpool
12612024-11-01T17:39:10.713Z root 1 0 0 16:38:31 ? 0:00 /sbin/init
12622024-11-01T17:39:10.713Z root 2 0 0 16:38:31 ? 0:01 pageout
12632024-11-01T17:39:10.713Z root 3 0 0 16:38:31 ? 0:03 fsflush
12642024-11-01T17:39:10.713Z root 11 1 0 16:38:31 ? 0:05 /lib/svc/bin/svc.configd
12652024-11-01T17:39:10.713Z root 9 1 0 16:38:31 ? 0:02 /lib/svc/bin/svc.startd
12662024-11-01T17:39:10.713Z root 631 1 0 16:38:35 ? 0:00 /sbin/dhcpagent
12672024-11-01T17:39:10.713Z root 83 1 0 16:38:33 ? 0:00 /lib/inet/in.mpathd
12682024-11-01T17:39:10.713Z root 320 288 0 16:38:34 ? 0:00 /usr/lib/hal/hald-addon-network-discovery
12692024-11-01T17:39:10.713Z root 894 9 0 16:38:41 ? 0:00 /usr/lib/saf/sac -t 300
12702024-11-01T17:39:10.713Z root 908 1 0 16:38:41 ? 0:00 /usr/lib/hotplugd
12712024-11-01T17:39:10.713Z netadm 54 1 0 16:38:33 ? 0:00 /lib/inet/ipmgmtd
12722024-11-01T17:39:10.713Z dladm 52 1 0 16:38:33 ? 0:00 /sbin/dlmgmtd
12732024-11-01T17:39:10.713Z netcfg 50 1 0 16:38:33 ? 0:00 /lib/inet/netcfgd
12742024-11-01T17:39:10.713Z root 265 1 0 16:38:33 ? 0:00 /usr/lib/hal/hald --daemon=yes
12752024-11-01T17:39:10.713Z root 134 1 0 16:38:33 ? 0:00 /usr/lib/sysevent/syseventd
12762024-11-01T17:39:10.713Z root 143 1 0 16:38:33 ? 0:00 devfsadmd
12772024-11-01T17:39:10.713Z root 133 1 0 16:38:33 ? 0:00 /usr/lib/power/powerd
12782024-11-01T17:39:10.713Z root 288 265 0 16:38:34 ? 0:00 hald-runner
12792024-11-01T17:39:10.713Z root 192 1 0 16:38:33 ? 0:00 /usr/lib/dbus-daemon --system
12802024-11-01T17:39:10.713Z root 853 1 0 16:38:41 ? 0:01 /usr/sbin/nscd
12812024-11-01T17:39:10.713Z root 1036 1 0 16:38:43 ? 0:00 /usr/perl5/bin/perl /usr/lib/intrd
12822024-11-01T17:39:10.713Z root 331 288 0 16:38:34 ? 0:00 /usr/lib/hal/hald-addon-acpi
12832024-11-01T17:39:10.713Z root 324 288 0 16:38:34 ? 0:00 /usr/lib/hal/hald-addon-cpufreq
12842024-11-01T17:39:10.713Z root 430 1 0 16:38:34 ? 0:00 /usr/lib/zones/zonestatd
12852024-11-01T17:39:10.713Z root 821 1 0 16:38:41 ? 0:00 /usr/lib/picl/picld
12862024-11-01T17:39:10.713Z root 868 9 0 16:38:41 console 0:00 /usr/lib/saf/ttymon -g -d /dev/console -l console -m ldterm,ttcompat -h -p ip-1
12872024-11-01T17:39:10.713Z root 939 1 0 16:38:41 ? 0:00 /usr/lib/autofs/automountd
12882024-11-01T17:39:10.713Z root 912 1 0 16:38:41 ? 0:00 /usr/sbin/cron
12892024-11-01T17:39:10.714Z root 433 1 0 16:38:34 ? 0:00 /usr/lib/pfexecd
12902024-11-01T17:39:10.714Z root 936 1 0 16:38:41 ? 0:00 /usr/lib/inet/in.ndpd
12912024-11-01T17:39:10.714Z root 941 939 0 16:38:41 ? 0:00 /usr/lib/autofs/automountd
12922024-11-01T17:39:10.714Z root 867 1 0 16:38:41 ? 0:00 /usr/lib/utmpd
12932024-11-01T17:39:10.714Z build 1163 1137 0 16:39:09 ? 0:02 mpstat -T d 1
12942024-11-01T17:39:10.714Z root 901 894 0 16:38:41 ? 0:00 /usr/lib/saf/ttymon
12952024-11-01T17:39:10.714Z root 1082 9 0 16:38:44 ? 0:00 /usr/bin/ctrun -l child -o noorphan,regent /opt/buildomat/lib/agent run
12962024-11-01T17:39:10.714Z build 1164 1137 0 16:39:09 ? 0:02 vmstat -T d -p 1
12972024-11-01T17:39:10.714Z build 1162 1137 0 16:39:09 ? 0:02 iostat -T d -xn 1
12982024-11-01T17:39:10.714Z root 1011 1 0 16:38:42 ? 0:00 /usr/lib/inet/inetd start
12992024-11-01T17:39:10.714Z root 969 1 0 16:38:42 ? 0:00 /usr/lib/fm/fmd/fmd
13002024-11-01T17:39:10.714Z build 1156 1137 0 16:39:09 ? 0:00 /bin/bash /tmp/YpfoCv7pZ2wpN1eL.sh
13012024-11-01T17:39:10.714Z daemon 924 1 0 16:38:41 ? 0:00 /usr/sbin/rpcbind
13022024-11-01T17:39:10.714Z root 1083 1082 0 16:38:44 ? 0:03 /opt/buildomat/lib/agent run
13032024-11-01T17:39:10.714Z root 994 1 0 16:38:42 ? 0:00 /usr/sbin/sshd
13042024-11-01T17:39:10.714Z build 1137 1083 0 16:39:01 ? 0:00 /bin/bash /tmp/YpfoCv7pZ2wpN1eL.sh
13052024-11-01T17:39:10.714Z build 1161 1137 0 16:39:09 ? 0:12 prstat -d d -mLc 1
13062024-11-01T17:39:10.714Z build 2586 1156 0 17:39:09 ? 0:00 ps -ef
13072024-11-01T17:39:10.714Z root 1165 1137 0 16:39:09 ? 0:03 dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
13082024-11-01T17:39:10.714Z root 1166 1137 0 16:39:09 ? 0:02 dtrace -Z -s /input/build/work/scripts/upstairs_info.d
13092024-11-01T17:39:10.717Z build 1168 1137 0 16:39:09 ? 0:00 ptime -m bash /input/build/work/scripts/test_up.sh -N encrypted
13102024-11-01T17:39:10.717Z build 1169 1168 0 16:39:09 ? 0:02 bash /input/build/work/scripts/test_up.sh -N encrypted
13112024-11-01T17:39:10.717Z build 2584 1169 0 17:39:06 ? 0:00 sleep 5
13122024-11-01T17:39:10.717Z build 1247 1169 0 16:43:01 ? 0:01 /var/tmp/bins/dsc start --region-count 4 --output-dir /tmp/test_up-build/dsc --
13132024-11-01T17:39:10.717Z build 1251 1247 0 16:43:01 ? 0:00 /var/tmp/bins/crucible-downstairs run -p 8820 -d /var/tmp/test_up-build/8820 --
13142024-11-01T17:39:10.717Z build 1249 1247 0 16:43:01 ? 0:00 /var/tmp/bins/crucible-downstairs run -p 8810 -d /var/tmp/test_up-build/8810 --
13152024-11-01T17:39:10.717Z build 1250 1247 0 16:43:01 ? 0:00 /var/tmp/bins/crucible-downstairs run -p 8840 -d /var/tmp/test_up-build/8840 --
13162024-11-01T17:39:10.717Z build 1252 1247 0 16:43:01 ? 0:00 /var/tmp/bins/crucible-downstairs run -p 8830 -d /var/tmp/test_up-build/8830 --
13172024-11-01T17:39:10.717Z+ zfs list
13182024-11-01T17:39:10.720ZNAME USED AVAIL REFER MOUNTPOINT
13192024-11-01T17:39:10.720Zrpool 86.3G 155G 25K /rpool
13202024-11-01T17:39:10.720Zrpool/ROOT 4.12G 155G 24K legacy
13212024-11-01T17:39:10.720Zrpool/ROOT/685856f1 4.12G 155G 4.12G legacy
13222024-11-01T17:39:10.720Zrpool/dump 32.0G 155G 32.0G -
13232024-11-01T17:39:10.720Zrpool/home 25.5K 155G 25.5K /home
13242024-11-01T17:39:10.720Zrpool/input 377M 155G 377M /input
13252024-11-01T17:39:10.720Zrpool/swap 49.9G 205G 5.71M -
13262024-11-01T17:39:10.720Zrpool/work 24K 155G 24K /work
13272024-11-01T17:39:10.720Z+ kill 1137
13282024-11-01T17:39:12.264ZDownstairs client 1 not running, waiting for it
13292024-11-01T17:39:15.726Zexec warning: : child terminated by signal 15
13302024-11-01T17:39:15.726Zprocess exited: duration 3608145 ms, exit code 2147483647
13312024-11-01T17:39:15.726Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
13322024-11-01T17:39:17.325ZDownstairs client 1 not running, waiting for it
13332024-11-01T17:39:22.386ZDownstairs client 1 not running, waiting for it
13342024-11-01T17:39:27.447ZDownstairs client 1 not running, waiting for it
13352024-11-01T17:39:32.508ZDownstairs client 1 not running, waiting for it
13362024-11-01T17:39:37.568ZDownstairs client 1 not running, waiting for it
13372024-11-01T17:39:42.629ZDownstairs client 1 not running, waiting for it
13382024-11-01T17:39:47.689ZDownstairs client 1 not running, waiting for it
13392024-11-01T17:39:52.749ZDownstairs client 1 not running, waiting for it
13402024-11-01T17:39:57.810ZDownstairs client 1 not running, waiting for it
13412024-11-01T17:40:02.869ZDownstairs client 1 not running, waiting for it
13422024-11-01T17:40:07.929ZDownstairs client 1 not running, waiting for it
13432024-11-01T17:40:12.990ZDownstairs client 1 not running, waiting for it
13442024-11-01T17:40:15.738Zexec warning: : stdout descriptor may be held open by a background process; giving up!
13452024-11-01T17:40:15.738Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
13462024-11-01T17:40:15.747Zfound 14 output files
13472024-11-01T17:40:15.748Zuploading: /tmp/test_up-build/dsc-out.txt (234314 bytes)
13482024-11-01T17:40:16.766Zuploaded: /tmp/test_up-build/dsc-out.txt
13492024-11-01T17:40:16.771Zuploading: /tmp/test_up-build/test_up_out.txt (3698610 bytes)
13502024-11-01T17:40:17.837Zuploaded: /tmp/test_up-build/test_up_out.txt
13512024-11-01T17:40:17.837Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (2503 bytes)
13522024-11-01T17:40:18.847Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
13532024-11-01T17:40:18.850Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (2505 bytes)
13542024-11-01T17:40:19.859Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
13552024-11-01T17:40:19.861Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (2506 bytes)
13562024-11-01T17:40:20.867Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
13572024-11-01T17:40:20.871Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (2506 bytes)
13582024-11-01T17:40:21.877Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
13592024-11-01T17:40:21.880Zuploading: /tmp/debug/df.txt (1270 bytes)
13602024-11-01T17:40:22.886Zuploaded: /tmp/debug/df.txt
13612024-11-01T17:40:22.889Zuploading: /tmp/debug/dtrace.txt (1424117 bytes)
13622024-11-01T17:40:23.937Zuploaded: /tmp/debug/dtrace.txt
13632024-11-01T17:40:23.937Zuploading: /tmp/debug/iostat.txt (1103476 bytes)
13642024-11-01T17:40:23.979Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 1103476 -> 1105884
13652024-11-01T17:40:24.985Zuploaded: /tmp/debug/iostat.txt
13662024-11-01T17:40:24.987Zuploading: /tmp/debug/mpstat.txt (2676186 bytes)
13672024-11-01T17:40:25.058Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 2676186 -> 2682756
13682024-11-01T17:40:26.064Zuploaded: /tmp/debug/mpstat.txt
13692024-11-01T17:40:26.065Zuploading: /tmp/debug/paging.txt (483242 bytes)
13702024-11-01T17:40:26.089Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 483242 -> 484560
13712024-11-01T17:40:27.096Zuploaded: /tmp/debug/paging.txt
13722024-11-01T17:40:27.102Zuploading: /tmp/debug/prstat.txt (4797153 bytes)
13732024-11-01T17:40:27.197Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 4797153 -> 4811362
13742024-11-01T17:40:28.203Zuploaded: /tmp/debug/prstat.txt
13752024-11-01T17:40:28.206Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
13762024-11-01T17:40:29.213Zuploaded: /tmp/debug/psrinfo.txt
13772024-11-01T17:40:29.213Zuploading: /tmp/debug/upstairs-info.txt (40290 bytes)
13782024-11-01T17:40:30.224Zuploaded: /tmp/debug/upstairs-info.txt