01JD2NDMCTS07QC3RMH3SCSJ4A: test-up-unencrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01JD2NDY9WQ3792HYJAC1MGT9B

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-11-19T17:06:27.916Zjob dependencies complete; ready to run (waiting for 13 m 59 s)
22024-11-19T17:07:35.143Zjob assigned to worker 01JD2P7NAF3EVMVKZHF1HVZV4F [factory aws, i-07f8790d456b3d241] (queued for 1 m 7 s)
32024-11-19T17:07:43.350Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-11-19T17:07:44.512Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-11-19T17:07:44.512Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-11-19T17:07:45.570Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-11-19T17:07:45.570Zdownloading input: /input/build/work/bins/crutest.gz
82024-11-19T17:07:46.765Zdownloaded input: /input/build/work/bins/crutest.gz
92024-11-19T17:07:46.766Zdownloading input: /input/build/work/bins/dsc.gz
102024-11-19T17:07:47.446Zdownloaded input: /input/build/work/bins/dsc.gz
112024-11-19T17:07:47.450Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-11-19T17:07:47.470Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-11-19T17:07:47.470Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-11-19T17:07:47.488Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-11-19T17:07:47.489Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-11-19T17:07:47.509Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-11-19T17:07:47.509Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-11-19T17:07:47.528Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-11-19T17:07:47.528Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-11-19T17:07:47.549Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-11-19T17:07:47.549Zdownloading input: /input/build/work/scripts/test_up.sh
222024-11-19T17:07:47.567Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-11-19T17:07:47.567Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-11-19T17:07:47.588Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-11-19T17:07:47.590Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-11-19T17:07:48.208Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-11-19T17:07:48.208Zstarting task 0: "setup"
282024-11-19T17:07:48.220Z++ uname -s
292024-11-19T17:07:48.222Z+ kern=SunOS
302024-11-19T17:07:48.222Z+ case "$kern" in
312024-11-19T17:07:48.222Z+ groupadd -g 12345 build
322024-11-19T17:07:48.225Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-11-19T17:07:50.231Z+ zfs create -o mountpoint=/work rpool/work
342024-11-19T17:07:50.383Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-11-19T17:07:50.385Z+ home_fs=zfs
362024-11-19T17:07:50.385Z+ [[ zfs == autofs ]]
372024-11-19T17:07:50.385Z+ mkdir -p /home/build
382024-11-19T17:07:50.388Z+ chown build:build /home/build /work
392024-11-19T17:07:52.392Z+ chmod 0700 /home/build /work
402024-11-19T17:07:52.394Zprocess exited: duration 4184 ms, exit code 0
 
412024-11-19T17:07:52.399Zstarting task 1: "authentication"
422024-11-19T17:07:52.417Zprocess exited: duration 17 ms, exit code 0
 
432024-11-19T17:07:52.422Zstarting task 2: "build"
442024-11-19T17:07:52.427Z+ banner cores
452024-11-19T17:07:52.449Z
462024-11-19T17:07:52.449Z #### #### ##### ###### ####
472024-11-19T17:07:52.449Z # # # # # # # #
482024-11-19T17:07:52.449Z # # # # # ##### ####
492024-11-19T17:07:52.449Z # # # ##### # #
502024-11-19T17:07:52.449Z # # # # # # # # #
512024-11-19T17:07:52.449Z #### #### # # ###### ####
522024-11-19T17:07:52.449Z
532024-11-19T17:07:52.452Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-11-19T17:07:52.458Z+ echo 'input bins dir contains:'
552024-11-19T17:07:52.461Zinput bins dir contains:
562024-11-19T17:07:52.461Z+ ls -ltr /input/build/work/bins
572024-11-19T17:07:52.461Ztotal 762763
582024-11-19T17:07:52.462Z-rw-r--r-- 1 root root 109276241 Nov 19 17:07 crucible-downstairs.gz
592024-11-19T17:07:52.462Z-rw-r--r-- 1 root root 105043485 Nov 19 17:07 crucible-hammer.gz
602024-11-19T17:07:52.462Z-rw-r--r-- 1 root root 109482608 Nov 19 17:07 crutest.gz
612024-11-19T17:07:52.462Z-rw-r--r-- 1 root root 66480669 Nov 19 17:07 dsc.gz
622024-11-19T17:07:52.462Z+ banner unpack
632024-11-19T17:07:52.465Z
642024-11-19T17:07:52.465Z # # # # ##### ## #### # #
652024-11-19T17:07:52.465Z # # ## # # # # # # # # #
662024-11-19T17:07:52.465Z # # # # # # # # # # ####
672024-11-19T17:07:52.465Z # # # # # ##### ###### # # #
682024-11-19T17:07:52.465Z # # # ## # # # # # # #
692024-11-19T17:07:52.465Z #### # # # # # #### # #
702024-11-19T17:07:52.465Z
712024-11-19T17:07:52.465Z+ mkdir -p /var/tmp/bins
722024-11-19T17:07:52.467Z+ for t in "$input/bins/"*.gz
732024-11-19T17:07:52.467Z++ basename /input/build/work/bins/crucible-downstairs.gz
742024-11-19T17:07:52.467Z+ b=crucible-downstairs.gz
752024-11-19T17:07:52.467Z+ b=crucible-downstairs
762024-11-19T17:07:52.468Z+ gunzip
772024-11-19T17:07:54.714Z+ chmod +x /var/tmp/bins/crucible-downstairs
782024-11-19T17:07:54.717Z+ for t in "$input/bins/"*.gz
792024-11-19T17:07:54.719Z++ basename /input/build/work/bins/crucible-hammer.gz
802024-11-19T17:07:54.719Z+ b=crucible-hammer.gz
812024-11-19T17:07:54.719Z+ b=crucible-hammer
822024-11-19T17:07:54.720Z+ gunzip
832024-11-19T17:07:56.874Z+ chmod +x /var/tmp/bins/crucible-hammer
842024-11-19T17:07:56.876Z+ for t in "$input/bins/"*.gz
852024-11-19T17:07:56.879Z++ basename /input/build/work/bins/crutest.gz
862024-11-19T17:07:56.879Z+ b=crutest.gz
872024-11-19T17:07:56.879Z+ b=crutest
882024-11-19T17:07:56.879Z+ gunzip
892024-11-19T17:07:59.191Z+ chmod +x /var/tmp/bins/crutest
902024-11-19T17:07:59.193Z+ for t in "$input/bins/"*.gz
912024-11-19T17:07:59.196Z++ basename /input/build/work/bins/dsc.gz
922024-11-19T17:07:59.198Z+ b=dsc.gz
932024-11-19T17:07:59.198Z+ b=dsc
942024-11-19T17:07:59.198Z+ gunzip
952024-11-19T17:08:00.586Z+ chmod +x /var/tmp/bins/dsc
962024-11-19T17:08:00.589Z+ export BINDIR=/var/tmp/bins
972024-11-19T17:08:00.589Z+ BINDIR=/var/tmp/bins
982024-11-19T17:08:00.589Z+ jobpid=1139
992024-11-19T17:08:00.591Z+ echo 'Setup debug logging'
1002024-11-19T17:08:00.591ZSetup debug logging
1012024-11-19T17:08:00.592Z+ mkdir /tmp/debug
1022024-11-19T17:08:00.592Z+ sleep 7200
1032024-11-19T17:08:00.594Z+ psrinfo -v
1042024-11-19T17:08:00.596Z+ df -h
1052024-11-19T17:08:00.623Z+ prstat -d d -mLc 1
1062024-11-19T17:08:00.626Z+ iostat -T d -xn 1
1072024-11-19T17:08:00.626Z+ mpstat -T d 1
1082024-11-19T17:08:00.626Z+ vmstat -T d -p 1
1092024-11-19T17:08:00.628Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1102024-11-19T17:08:00.628Z+ banner test_up_unencrypted
1112024-11-19T17:08:00.628Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1122024-11-19T17:08:00.631Z
1132024-11-19T17:08:00.631Z ##### ###### #### ##### # # ##### # # # #
1142024-11-19T17:08:00.631Z # # # # # # # # # # ## #
1152024-11-19T17:08:00.631Z # ##### #### # # # # # # # # # #
1162024-11-19T17:08:00.631Z # # # # # # ##### # # # # #
1172024-11-19T17:08:00.631Z # # # # # # # # # # # ##
1182024-11-19T17:08:00.631Z # ###### #### # ####### #### # ####### #### # #
1192024-11-19T17:08:00.631Z
1202024-11-19T17:08:00.631Z+ ptime -m bash /input/build/work/scripts/test_up.sh -N unencrypted
1212024-11-19T17:08:00.640Z/input/build/work
1222024-11-19T17:08:00.675ZTurn off color for downstairs dump
1232024-11-19T17:08:00.677Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1242024-11-19T17:08:00.678ZCreating 3 downstairs regions
1252024-11-19T17:08:01.019ZStarting 3 downstairs
1262024-11-19T17:08:01.021Zdsc started at PID: 1184
1272024-11-19T17:08:06.194ZDisable automatic restart on all downstairs
1282024-11-19T17:08:06.245Z
1292024-11-19T17:08:06.245ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1302024-11-19T17:08:06.250ZRunning test: span
1312024-11-19T17:08:10.391ZCompleted test: span
1322024-11-19T17:08:15.398ZRunning test: big
1332024-11-19T17:08:19.578ZCompleted test: big
1342024-11-19T17:08:24.585ZRunning test: dep
1352024-11-19T17:08:37.251ZCompleted test: dep
1362024-11-19T17:08:42.259ZRunning test: balloon
1372024-11-19T17:08:51.872ZCompleted test: balloon
1382024-11-19T17:08:56.879ZRunning test: deactivate
1392024-11-19T17:09:51.117ZCompleted test: deactivate
1402024-11-19T17:09:56.124ZRunning hammer
1412024-11-19T17:09:59.309ZRun repair tests
1422024-11-19T17:09:59.313Z/var/tmp/bins/crutest fill -g 31 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1432024-11-19T17:09:59.372Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.828581213Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"task":"crutest"}
1442024-11-19T17:09:59.375Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.829174778Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"task":"crutest"}
1452024-11-19T17:09:59.375Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.83086436Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"task":"crutest"}
1462024-11-19T17:09:59.375Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.831069039Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221}
1472024-11-19T17:09:59.375Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"775749f049384d6df9adbcf0df5fb1eb2a3a0b7e\",\n git_commit_timestamp: \"2024-11-19T16:38:47.000000000Z\",\n git_branch: \"mkeeter/early-io-rejection\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.831110626Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221}
1482024-11-19T17:09:59.375Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.831133495Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221}
1492024-11-19T17:09:59.375Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.831149624Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221}
1502024-11-19T17:09:59.378Z{"msg":"Crucible 3fc113e4-99e3-4d99-9a20-e3a5f875b95b has session id: 5f222b9b-c2d6-489d-8102-7e24a988734b","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.831597288Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1512024-11-19T17:09:59.378Z{"msg":"Upstairs opts: Upstairs UUID: 3fc113e4-99e3-4d99-9a20-e3a5f875b95b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.831658465Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1522024-11-19T17:09:59.378Z{"msg":"Crucible stats registered with UUID: 3fc113e4-99e3-4d99-9a20-e3a5f875b95b","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.831694412Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1532024-11-19T17:09:59.409Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.86532982Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"task":"crutest"}
1542024-11-19T17:09:59.409Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.865383017Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221}
1552024-11-19T17:09:59.412Z{"msg":"3fc113e4-99e3-4d99-9a20-e3a5f875b95b active request set","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.865419795Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1562024-11-19T17:09:59.412Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.865460353Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"client":"0","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1572024-11-19T17:09:59.412Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.865520109Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"client":"0","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1582024-11-19T17:09:59.412Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.865557037Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"client":"1","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1592024-11-19T17:09:59.412Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.865605664Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"client":"1","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1602024-11-19T17:09:59.412Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.865646922Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"client":"2","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1612024-11-19T17:09:59.412Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.865687159Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"client":"2","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1622024-11-19T17:09:59.412Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.865710188Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"":"io task","client":"2","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1632024-11-19T17:09:59.412Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.865754256Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"":"io task","client":"0","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1642024-11-19T17:09:59.412Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.865790053Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"":"io task","client":"1","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1652024-11-19T17:09:59.412Z{"msg":"ds_connection connected from Ok(127.0.0.1:58208)","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.865837321Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"":"io task","client":"2","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1662024-11-19T17:09:59.412Z{"msg":"ds_connection connected from Ok(127.0.0.1:44908)","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.866036849Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"":"io task","client":"1","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1672024-11-19T17:09:59.412Z{"msg":"ds_connection connected from Ok(127.0.0.1:46667)","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.866132304Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"":"io task","client":"0","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1682024-11-19T17:09:59.412Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.86653808Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"client":"1","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1692024-11-19T17:09:59.412Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.866604166Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"client":"1","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1702024-11-19T17:09:59.412Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.866732799Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"client":"2","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1712024-11-19T17:09:59.412Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.866802445Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"client":"2","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1722024-11-19T17:09:59.412Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.866859462Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"client":"0","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1732024-11-19T17:09:59.412Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.866920408Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"client":"0","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1742024-11-19T17:09:59.412Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.86723055Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"client":"1","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1752024-11-19T17:09:59.412Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.867271947Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"client":"2","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1762024-11-19T17:09:59.412Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.867388111Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"client":"0","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1772024-11-19T17:09:59.412Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.867605828Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"client":"1","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1782024-11-19T17:09:59.412Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.867677564Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"client":"1","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1792024-11-19T17:09:59.412Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.867726741Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1802024-11-19T17:09:59.412Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.867786738Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"client":"2","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1812024-11-19T17:09:59.412Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.867819216Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"client":"2","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1822024-11-19T17:09:59.412Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.867834495Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1832024-11-19T17:09:59.412Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.867860124Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"client":"0","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1842024-11-19T17:09:59.417Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.867874653Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"client":"0","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1852024-11-19T17:09:59.417Z{"msg":"[0]R flush_numbers: [1497, 1497, 1497, 1497, 1497]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.867896761Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"":"downstairs","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1862024-11-19T17:09:59.417Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.867911461Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"":"downstairs","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1872024-11-19T17:09:59.417Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.86792544Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"":"downstairs","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1882024-11-19T17:09:59.417Z{"msg":"[1]R flush_numbers: [1497, 1497, 1497, 1497, 1497]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.867946588Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"":"downstairs","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1892024-11-19T17:09:59.417Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.867960738Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"":"downstairs","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1902024-11-19T17:09:59.417Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.867974577Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"":"downstairs","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1912024-11-19T17:09:59.417Z{"msg":"[2]R flush_numbers: [1497, 1497, 1497, 1497, 1497]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.867994966Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"":"downstairs","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1922024-11-19T17:09:59.418Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.868009205Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"":"downstairs","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1932024-11-19T17:09:59.418Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.868022914Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"":"downstairs","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1942024-11-19T17:09:59.418Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.868036883Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"":"downstairs","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1952024-11-19T17:09:59.418Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.868050742Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"":"downstairs","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1962024-11-19T17:09:59.418Z{"msg":"Next flush: 1498","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.868063912Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"":"downstairs","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1972024-11-19T17:09:59.418Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.868084471Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"":"downstairs","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1982024-11-19T17:09:59.418Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.86809845Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
1992024-11-19T17:09:59.418Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time"Wait for a query_work_queue command to finish before sending IO
2002024-11-19T17:09:59.418Z:"2024-11-19T17:09:57.868110769Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"client":"0","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
2012024-11-19T17:09:59.418Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.868139267Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"client":"1","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
2022024-11-19T17:09:59.418Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.868152147Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"client":"2","":"downstairs","session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
2032024-11-19T17:09:59.418Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.868165536Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
2042024-11-19T17:09:59.418Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.868177985Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
2052024-11-19T17:09:59.418Z{"msg":"3fc113e4-99e3-4d99-9a20-e3a5f875b95b is now active with session: 5f222b9b-c2d6-489d-8102-7e24a988734b","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.868191434Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
2062024-11-19T17:09:59.418Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.868203813Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221,"session_id":"5f222b9b-c2d6-489d-8102-7e24a988734b"}
2072024-11-19T17:09:59.418Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:09:57.868216213Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1221}
2082024-11-19T17:09:59.418ZDisk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2092024-11-19T17:09:59.418ZFill test
2102024-11-19T17:09:59.421ZRead and Verify all blocks (0..50 range:false)
2112024-11-19T17:09:59.423ZWrote out file "/var/tmp/test_up-build/verify_file"
2122024-11-19T17:09:59.423ZCLIENT: Tests done. All submitted work has been ACK'd
2132024-11-19T17:09:59.423Z----------------------------------------------------------------
2142024-11-19T17:09:59.423Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2152024-11-19T17:09:59.423ZStates: Active Active Active
2162024-11-19T17:09:59.423ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2172024-11-19T17:09:59.423Z 1002 Acked Read 50 Done Done Done false
2182024-11-19T17:09:59.423Z STATES DS:0 DS:1 DS:2 TOTAL
2192024-11-19T17:09:59.424Z Sent 0 0 0 0
2202024-11-19T17:09:59.424Z Done 1 1 1 3
2212024-11-19T17:09:59.424Z Skipped 0 0 0 0
2222024-11-19T17:09:59.424Z Error 0 0 0 0
2232024-11-19T17:09:59.424ZLast Flush: 1001 1001 1001
2242024-11-19T17:09:59.424ZDownstairs last five completed: 1001 1000
2252024-11-19T17:09:59.424ZUpstairs last five completed: 1002 1001 1000
2262024-11-19T17:09:59.424ZCLIENT: Up:0 ds:1 act:3
2272024-11-19T17:10:03.427Z----------------------------------------------------------------
2282024-11-19T17:10:03.427Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2292024-11-19T17:10:03.427ZDownstairs last five completed: 1003 1002 1001 1000
2302024-11-19T17:10:03.427ZUpstairs last five completed: 1003 1002 1001 1000
2312024-11-19T17:10:03.431ZCLIENT: Up:0 ds:0 act:3
2322024-11-19T17:10:03.431ZCLIENT: All crucible jobs finished, exiting program
2332024-11-19T17:10:03.442ZRepair setup passed
2342024-11-19T17:10:03.444ZCopy the region for /var/tmp/test_up-build/8830
2352024-11-19T17:10:03.444Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2362024-11-19T17:10:03.446Z/var/tmp/bins/crutest fill -g 32 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2372024-11-19T17:10:03.506Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:01.961802757Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"task":"crutest"}
2382024-11-19T17:10:03.509Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:01.96243023Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"task":"crutest"}
2392024-11-19T17:10:03.509Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:01.964245055Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"task":"crutest"}
2402024-11-19T17:10:03.509Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:01.964443134Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227}
2412024-11-19T17:10:03.509Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"775749f049384d6df9adbcf0df5fb1eb2a3a0b7e\",\n git_commit_timestamp: \"2024-11-19T16:38:47.000000000Z\",\n git_branch: \"mkeeter/early-io-rejection\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:01.964490811Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227}
2422024-11-19T17:10:03.509Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:01.964518629Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227}
2432024-11-19T17:10:03.510Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:01.964537588Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227}
2442024-11-19T17:10:03.510Z{"msg":"Crucible 20b04d46-dd36-4f4b-aa78-76df387d1ed0 has session id: 5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:01.964927035Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2452024-11-19T17:10:03.510Z{"msg":"Upstairs opts: Upstairs UUID: 20b04d46-dd36-4f4b-aa78-76df387d1ed0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:01.965001591Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2462024-11-19T17:10:03.510Z{"msg":"Crucible stats registered with UUID: 20b04d46-dd36-4f4b-aa78-76df387d1ed0","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:01.965049218Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2472024-11-19T17:10:03.543Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:01.998824378Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"task":"crutest"}
2482024-11-19T17:10:03.543Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:01.998897754Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227}
2492024-11-19T17:10:03.546Z{"msg":"20b04d46-dd36-4f4b-aa78-76df387d1ed0 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:01.998946771Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2502024-11-19T17:10:03.546Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:01.998993988Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"client":"0","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2512024-11-19T17:10:03.546Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:01.999042395Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"client":"0","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2522024-11-19T17:10:03.546Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:01.999090543Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"client":"1","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2532024-11-19T17:10:03.547Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:01.999126111Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"client":"1","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2542024-11-19T17:10:03.547Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:01.999172818Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"client":"2","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2552024-11-19T17:10:03.547Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:01.999211886Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"client":"2","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2562024-11-19T17:10:03.547Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:01.999257733Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"":"io task","client":"2","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2572024-11-19T17:10:03.547Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:01.99931274Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"":"io task","client":"0","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2582024-11-19T17:10:03.547Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:01.999357747Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"":"io task","client":"1","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2592024-11-19T17:10:03.547Z{"msg":"ds_connection connected from Ok(127.0.0.1:41670)","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:01.999397755Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"":"io task","client":"1","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2602024-11-19T17:10:03.547Z{"msg":"ds_connection connected from Ok(127.0.0.1:35830)","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:01.999430233Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"":"io task","client":"0","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2612024-11-19T17:10:03.547Z{"msg":"ds_connection connected from Ok(127.0.0.1:48695)","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:01.999455982Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"":"io task","client":"2","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2622024-11-19T17:10:03.547Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.000048957Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"client":"1","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2632024-11-19T17:10:03.547Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.000101414Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"client":"1","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2642024-11-19T17:10:03.547Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.000135812Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"client":"2","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2652024-11-19T17:10:03.547Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.00016369Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"client":"2","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2662024-11-19T17:10:03.547Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.000200048Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"client":"0","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2672024-11-19T17:10:03.547Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.000217207Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"client":"0","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2682024-11-19T17:10:03.547Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.000834771Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"client":"1","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2692024-11-19T17:10:03.547Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.000894938Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"client":"2","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2702024-11-19T17:10:03.547Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.000934016Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"client":"0","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2712024-11-19T17:10:03.547Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.001222929Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"client":"2","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2722024-11-19T17:10:03.547Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.001262107Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"client":"2","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2732024-11-19T17:10:03.548Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.001279046Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2742024-11-19T17:10:03.548Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.001340962Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"client":"1","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2752024-11-19T17:10:03.548Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.00137641Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"client":"1","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2762024-11-19T17:10:03.548Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.001391359Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2772024-11-19T17:10:03.548Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.001405058Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"client":"0","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2782024-11-19T17:10:03.548Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.001426737Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"client":"0","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2792024-11-19T17:10:03.548Z{"msg":"[0]R flush_numbers: [1498, 1498, 1498, 1498, 1498]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.001441246Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"":"downstairs","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2802024-11-19T17:10:03.548Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.001462945Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"":"downstairs","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2812024-11-19T17:10:03.548Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.001477004Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"":"downstairs","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2822024-11-19T17:10:03.548Z{"msg":"[1]R flush_numbers: [1498, 1498, 1498, 1498, 1498]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.001490923Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"":"downstairs","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2832024-11-19T17:10:03.548Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.001504943Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"":"downstairs","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2842024-11-19T17:10:03.551Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.001518502Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"":"downstairs","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2852024-11-19T17:10:03.551Z{"msg":"[2]R flush_numbers: [1498, 1498, 1498, 1498, 1498]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.00155289Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"":"downstairs","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2862024-11-19T17:10:03.551Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
2872024-11-19T17:10:03.551Z,"time":"2024-11-19T17:10:02.001574108Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"":"downstairs","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2882024-11-19T17:10:03.551Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.001592927Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"":"downstairs","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2892024-11-19T17:10:03.551Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.001617296Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"":"downstairs","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2902024-11-19T17:10:03.551Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.001637135Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"":"downstairs","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2912024-11-19T17:10:03.551Z{"msg":"Next flush: 1499","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.001669973Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"":"downstairs","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2922024-11-19T17:10:03.552Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.001686572Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"":"downstairs","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2932024-11-19T17:10:03.552Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.00172028Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2942024-11-19T17:10:03.552Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.001736169Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"client":"0","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2952024-11-19T17:10:03.552Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.001767367Z","hostname":"Disk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2962024-11-19T17:10:03.552Zw-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"client":"1","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2972024-11-19T17:10:03.552Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.001802575Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"client":"2","":"downstairs","session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2982024-11-19T17:10:03.552Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.001836093Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
2992024-11-19T17:10:03.552Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.001867891Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
3002024-11-19T17:10:03.552Z{"msg":"20b04d46-dd36-4f4b-aa78-76df387d1ed0 is now active with session: 5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.00188617Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
3012024-11-19T17:10:03.552Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:02.001915319Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227,"session_id":"5bd5bcb4-df1b-4f0b-a9be-31e7117d9cdf"}
3022024-11-19T17:10:03.552Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30Loading write count information from file "/var/tmp/test_up-build/verify_file"
3032024-11-19T17:10:03.552Z,"time":"2024-11-19T17:10:02.001933168Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1227}
3042024-11-19T17:10:03.552ZFill test
3052024-11-19T17:10:03.552ZRead and Verify all blocks (0..50 range:false)
3062024-11-19T17:10:03.557ZWrote out file "/var/tmp/test_up-build/verify_file"
3072024-11-19T17:10:03.557ZCLIENT: Tests done. All submitted work has been ACK'd
3082024-11-19T17:10:03.560Z----------------------------------------------------------------
3092024-11-19T17:10:03.560Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
3102024-11-19T17:10:03.560ZStates: Active Active Active
3112024-11-19T17:10:03.560ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
3122024-11-19T17:10:03.560Z 1002 Acked Read 50 Done Done Done false
3132024-11-19T17:10:03.560Z STATES DS:0 DS:1 DS:2 TOTAL
3142024-11-19T17:10:03.560Z Sent 0 0 0 0
3152024-11-19T17:10:03.561Z Done 1 1 1 3
3162024-11-19T17:10:03.561Z Skipped 0 0 0 0
3172024-11-19T17:10:03.561Z Error 0 0 0 0
3182024-11-19T17:10:03.561ZLast Flush: 1001 1001 1001
3192024-11-19T17:10:03.561ZDownstairs last five completed: 1001 1000
3202024-11-19T17:10:03.561ZUpstairs last five completed: 1002 1001 1000
3212024-11-19T17:10:03.561ZCLIENT: Up:0 ds:1 act:3
3222024-11-19T17:10:07.559Z----------------------------------------------------------------
3232024-11-19T17:10:07.559Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
3242024-11-19T17:10:07.559ZDownstairs last five completed: 1003 1002 1001 1000
3252024-11-19T17:10:07.559ZUpstairs last five completed: 1003 1002 1001 1000
3262024-11-19T17:10:07.559ZCLIENT: Up:0 ds:0 act:3
3272024-11-19T17:10:07.559ZCLIENT: All crucible jobs finished, exiting program
3282024-11-19T17:10:07.566ZRepair part 1 passed
3292024-11-19T17:10:07.568Z
3302024-11-19T17:10:07.568ZKill the current downstairs
3312024-11-19T17:10:07.728ZDownstairs 2 stopped
3322024-11-19T17:10:07.728Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
3332024-11-19T17:10:07.730ZNow put back the original so we have a mismatch
3342024-11-19T17:10:07.730Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
3352024-11-19T17:10:07.732ZRestart downstairs with old directory
3362024-11-19T17:10:07.785Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
3372024-11-19T17:10:07.801ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3382024-11-19T17:10:07.803Z 0 000-009 32 32 31 1499 1499 1498 F F F <---
3392024-11-19T17:10:07.803Z 1 010-019 32 32 31 1499 1499 1498 F F F <---
3402024-11-19T17:10:07.803Z 2 020-029 32 32 31 1499 1499 1498 F F F <---
3412024-11-19T17:10:07.803Z 3 030-039 32 32 31 1499 1499 1498 F F F <---
3422024-11-19T17:10:07.803Z 4 040-049 32 32 31 1499 1499 1498 F F F <---
3432024-11-19T17:10:07.803ZMax gen: 32, Max flush: 1499
3442024-11-19T17:10:07.803ZError: Difference in extent metadata found!
3452024-11-19T17:10:07.806Zdump test found error as expected
3462024-11-19T17:10:07.806Z
3472024-11-19T17:10:07.806Z
3482024-11-19T17:10:07.806Z/var/tmp/bins/crutest verify --range -g 33 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
3492024-11-19T17:10:07.863Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.318194757Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"task":"crutest"}
3502024-11-19T17:10:07.865Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.318761485Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"task":"crutest"}
3512024-11-19T17:10:07.866Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.320460006Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"task":"crutest"}
3522024-11-19T17:10:07.866Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.320652285Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235}
3532024-11-19T17:10:07.866Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"775749f049384d6df9adbcf0df5fb1eb2a3a0b7e\",\n git_commit_timestamp: \"2024-11-19T16:38:47.000000000Z\",\n git_branch: \"mkeeter/early-io-rejection\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.320692002Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235}
3542024-11-19T17:10:07.866Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.320708012Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235}
3552024-11-19T17:10:07.866Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.32072294Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235}
3562024-11-19T17:10:07.868Z{"msg":"Crucible f23595e5-364d-4150-ba80-5fa9a2c99d60 has session id: 9320b0f0-18cb-4bf9-ad35-f17ad5dacbae","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.321180044Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3572024-11-19T17:10:07.868Z{"msg":"Upstairs opts: Upstairs UUID: f23595e5-364d-4150-ba80-5fa9a2c99d60, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.321232041Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3582024-11-19T17:10:07.868Z{"msg":"Crucible stats registered with UUID: f23595e5-364d-4150-ba80-5fa9a2c99d60","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.321269449Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3592024-11-19T17:10:07.899Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.355064727Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"task":"crutest"}
3602024-11-19T17:10:07.899Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.355127054Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235}
3612024-11-19T17:10:07.902Z{"msg":"f23595e5-364d-4150-ba80-5fa9a2c99d60 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.355176031Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3622024-11-19T17:10:07.902Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.355223618Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"client":"0","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3632024-11-19T17:10:07.902Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.355286255Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"client":"0","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3642024-11-19T17:10:07.902Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.355325882Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"client":"1","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3652024-11-19T17:10:07.902Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.35535903Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"client":"1","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3662024-11-19T17:10:07.902Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.355420567Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"client":"2","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3672024-11-19T17:10:07.902Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.355453055Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"client":"2","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3682024-11-19T17:10:07.902Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.355500372Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3692024-11-19T17:10:07.902Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.355575308Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"":"io task","client":"0","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3702024-11-19T17:10:07.902Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.355615715Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"":"io task","client":"1","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3712024-11-19T17:10:07.902Z{"msg":"ds_connection connected from Ok(127.0.0.1:64882)","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.35571095Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"":"io task","client":"0","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3722024-11-19T17:10:07.902Z{"msg":"ds_connection connected from Ok(127.0.0.1:47570)","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.355747738Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"":"io task","client":"1","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3732024-11-19T17:10:07.902Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-11-19T17:10:06.355770537Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3742024-11-19T17:10:07.902Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-11-19T17:10:06.355802085Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3752024-11-19T17:10:07.902Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-11-19T17:10:06.355852592Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3762024-11-19T17:10:07.902Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.35588306Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3772024-11-19T17:10:07.902Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.356333404Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"client":"1","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3782024-11-19T17:10:07.903Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.35640283Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"client":"1","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3792024-11-19T17:10:07.903Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.356469066Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"client":"0","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3802024-11-19T17:10:07.903Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.356487605Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"client":"0","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3812024-11-19T17:10:07.903Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.356947448Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"client":"0","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3822024-11-19T17:10:07.903Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.357107589Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"client":"1","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3832024-11-19T17:10:07.903Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.357310207Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"client":"0","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3842024-11-19T17:10:07.903Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.357349255Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"client":"0","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3852024-11-19T17:10:07.903Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.357366494Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3862024-11-19T17:10:07.906Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.357393262Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"client":"1","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3872024-11-19T17:10:07.906Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.357416651Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"client":"1","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3882024-11-19T17:10:07.906Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:06.35743094Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3892024-11-19T17:10:17.902Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.356719527Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3902024-11-19T17:10:17.902Z{"msg":"ds_connection connected from Ok(127.0.0.1:59293)","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.35684845Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3912024-11-19T17:10:17.906Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.35822584Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"client":"2","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3922024-11-19T17:10:17.906Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.358268577Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"client":"2","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3932024-11-19T17:10:17.906Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.358849444Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"client":"2","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3942024-11-19T17:10:17.906Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.359193774Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"client":"2","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3952024-11-19T17:10:17.906Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.359228752Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"client":"2","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3962024-11-19T17:10:17.906Z{"msg":"[0]R flush_numbers: [1499, 1499, 1499, 1499, 1499]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.359245281Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3972024-11-19T17:10:17.906Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.35926498Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3982024-11-19T17:10:17.906Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.359297038Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
3992024-11-19T17:10:17.906Z{"msg":"[1]R flush_numbers: [1499, 1499, 1499, 1499, 1499]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.359313987Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4002024-11-19T17:10:17.906Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.359346925Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4012024-11-19T17:10:17.906Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.359363764Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4022024-11-19T17:10:17.906Z{"msg":"[2]R flush_numbers: [1498, 1498, 1498, 1498, 1498]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.359396372Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4032024-11-19T17:10:17.906Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.359412311Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4042024-11-19T17:10:17.906Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.359444109Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4052024-11-19T17:10:17.906Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.359459948Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4062024-11-19T17:10:17.906Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.359491766Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4072024-11-19T17:10:17.906Z{"msg":"Next flush: 1500","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.359507816Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4082024-11-19T17:10:17.906Z{"msg":"Extent 0 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.359531204Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4092024-11-19T17:10:17.906Z{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.359561532Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4102024-11-19T17:10:17.906Z{"msg":"extent:0 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.359578601Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4112024-11-19T17:10:17.906Z{"msg":"extent:0 flush: 1499 1499 1498 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.35959637Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4122024-11-19T17:10:17.906Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.359628129Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4132024-11-19T17:10:17.906Z{"msg":"extent:0 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.359645757Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4142024-11-19T17:10:17.906Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.359679226Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4152024-11-19T17:10:17.906Z{"msg":"find dest for source 0 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.359696405Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4162024-11-19T17:10:17.907Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.359729723Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4172024-11-19T17:10:17.907Z{"msg":"Extent 1 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.359756791Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4182024-11-19T17:10:17.907Z{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.359790349Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4192024-11-19T17:10:17.907Z{"msg":"extent:1 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.359809458Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4202024-11-19T17:10:17.907Z{"msg":"extent:1 flush: 1499 1499 1498 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.359842256Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4212024-11-19T17:10:17.907Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.359859375Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4222024-11-19T17:10:17.910Z{"msg":"extent:1 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.359893693Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4232024-11-19T17:10:17.910Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.359921952Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4242024-11-19T17:10:17.911Z{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.359964249Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4252024-11-19T17:10:17.911Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.360010746Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4262024-11-19T17:10:17.911Z{"msg":"Extent 2 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.360042854Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4272024-11-19T17:10:17.911Z{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.360077863Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4282024-11-19T17:10:17.911Z{"msg":"extent:2 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.360110001Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4292024-11-19T17:10:17.911Z{"msg":"extent:2 flush: 1499 1499 1498 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.360133529Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4302024-11-19T17:10:17.911Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.360167077Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4312024-11-19T17:10:17.911Z{"msg":"extent:2 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.360203685Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4322024-11-19T17:10:17.911Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.360234483Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4332024-11-19T17:10:17.911Z{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.360274931Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4342024-11-19T17:10:17.911Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.360307319Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4352024-11-19T17:10:17.911Z{"msg":"Extent 3 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.360323958Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4362024-11-19T17:10:17.911Z{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.360365726Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4372024-11-19T17:10:17.911Z{"msg":"extent:3 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.360392344Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4382024-11-19T17:10:17.911Z{"msg":"extent:3 flush: 1499 1499 1498 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.360422942Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4392024-11-19T17:10:17.911Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.360454501Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4402024-11-19T17:10:17.912Z{"msg":"extent:3 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.360502338Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4412024-11-19T17:10:17.912Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.360538736Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4422024-11-19T17:10:17.912Z{"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.360588223Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4432024-11-19T17:10:17.912Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.360608042Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4442024-11-19T17:10:17.912Z{"msg":"Extent 4 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.360623861Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4452024-11-19T17:10:17.912Z{"msg":"First source client ID for extent 4","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.360647249Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4462024-11-19T17:10:17.912Z{"msg":"extent:4 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.360662878Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4472024-11-19T17:10:17.912Z{"msg":"extent:4 flush: 1499 1499 1498 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.360694977Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4482024-11-19T17:10:17.912Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.360725005Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4492024-11-19T17:10:17.912Z{"msg":"extent:4 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.360741484Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4502024-11-19T17:10:17.912Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.360758403Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4512024-11-19T17:10:17.912Z{"msg":"find dest for source 0 for extent at index 4","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.360780482Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4522024-11-19T17:10:17.912Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.360797491Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4532024-11-19T17:10:17.912Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.360834879Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"client":"0","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4542024-11-19T17:10:17.912Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.360877696Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"client":"1","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4552024-11-19T17:10:17.912Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.360923103Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"client":"2","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4562024-11-19T17:10:17.913Z{"msg":"starting reconciliation 337c98b2-65c1-4631-974a-1b1c021261c4: found 5 extents that need repair","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.360962731Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4572024-11-19T17:10:17.913Z{"msg":"Full repair list: {ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.361006089Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4582024-11-19T17:10:17.913Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 1500, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 1500, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 1500, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 1500, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 1500, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.3611486Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4592024-11-19T17:10:17.913Z{"msg":"no Nexus client from DNS, aborting start notification","v":0,"name":"crucible","level":50,"time":"2024-11-19T17:10:16.361188648Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"reconcile":"337c98b2-65c1-4631-974a-1b1c021261c4","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4602024-11-19T17:10:17.913Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-19T17:10:16.361214026Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"reconcile":"337c98b2-65c1-4631-974a-1b1c021261c4","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4612024-11-19T17:10:17.913Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-19T17:10:16.361249044Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"reconcile":"337c98b2-65c1-4631-974a-1b1c021261c4","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4622024-11-19T17:10:17.913Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.361281773Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"client":"2","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4632024-11-19T17:10:17.914Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-19T17:10:16.361312021Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"reconcile":"337c98b2-65c1-4631-974a-1b1c021261c4","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4642024-11-19T17:10:17.916Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-19T17:10:16.36804374Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"reconcile":"337c98b2-65c1-4631-974a-1b1c021261c4","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4652024-11-19T17:10:17.916Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-19T17:10:16.368454776Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"reconcile":"337c98b2-65c1-4631-974a-1b1c021261c4","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4662024-11-19T17:10:17.916Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-19T17:10:16.368704782Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"reconcile":"337c98b2-65c1-4631-974a-1b1c021261c4","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4672024-11-19T17:10:17.916Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.368968096Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"client":"2","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4682024-11-19T17:10:17.916Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-19T17:10:16.369020773Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"reconcile":"337c98b2-65c1-4631-974a-1b1c021261c4","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4692024-11-19T17:10:17.918Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-19T17:10:16.372136363Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"reconcile":"337c98b2-65c1-4631-974a-1b1c021261c4","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4702024-11-19T17:10:17.919Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-19T17:10:16.372472203Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"reconcile":"337c98b2-65c1-4631-974a-1b1c021261c4","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4712024-11-19T17:10:17.919Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-19T17:10:16.372724789Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"reconcile":"337c98b2-65c1-4631-974a-1b1c021261c4","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4722024-11-19T17:10:17.919Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.372991773Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"client":"2","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4732024-11-19T17:10:17.919Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-19T17:10:16.37303839Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"reconcile":"337c98b2-65c1-4631-974a-1b1c021261c4","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4742024-11-19T17:10:17.921Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-19T17:10:16.375858047Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"reconcile":"337c98b2-65c1-4631-974a-1b1c021261c4","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4752024-11-19T17:10:17.923Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-19T17:10:16.376216366Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"reconcile":"337c98b2-65c1-4631-974a-1b1c021261c4","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4762024-11-19T17:10:17.923Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-19T17:10:16.37649174Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"reconcile":"337c98b2-65c1-4631-974a-1b1c021261c4","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4772024-11-19T17:10:17.923Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.376801362Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"client":"2","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4782024-11-19T17:10:17.923Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-19T17:10:16.37683892Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"reconcile":"337c98b2-65c1-4631-974a-1b1c021261c4","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4792024-11-19T17:10:17.926Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-19T17:10:16.379613209Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"reconcile":"337c98b2-65c1-4631-974a-1b1c021261c4","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4802024-11-19T17:10:17.926Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-19T17:10:16.379973368Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"reconcile":"337c98b2-65c1-4631-974a-1b1c021261c4","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4812024-11-19T17:10:17.926Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-19T17:10:16.380248382Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"reconcile":"337c98b2-65c1-4631-974a-1b1c021261c4","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4822024-11-19T17:10:17.926Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.380496487Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"client":"2","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4832024-11-19T17:10:17.926Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-19T17:10:16.380583672Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"reconcile":"337c98b2-65c1-4631-974a-1b1c021261c4","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4842024-11-19T17:10:17.928Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-19T17:10:16.383433807Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"reconcile":"337c98b2-65c1-4631-974a-1b1c021261c4","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4852024-11-19T17:10:17.931Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.383680353Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4862024-11-19T17:10:17.931Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.38372266Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"client":"0","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4872024-11-19T17:10:17.931Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.383741729Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"client":"1","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4882024-11-19T17:10:17.931Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.383755898Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"client":"2","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4892024-11-19T17:10:17.931Z{"Wait for a query_work_queue command to finish before sending IO
4902024-11-19T17:10:17.931Zmsg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.383788216Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4912024-11-19T17:10:17.931Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.383823444Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4922024-11-19T17:10:17.931Z{"msg":"f23595e5-364d-4150-ba80-5fa9a2c99d60 is now active with session: 9320b0f0-18cb-4bf9-ad35-f17ad5dacbae","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.383864122Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4932024-11-19T17:10:17.932Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2024-11-19T17:10:16.38390308Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235}
4942024-11-19T17:10:17.932Z{"msg":"no Nexus client from DNS, aborting finish notification","v":0,"name":"crucible","level":50,"time":"2024-11-19T17:10:16.383929538Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235,"reconcile":"337c98b2-65c1-4631-974a-1b1c021261c4","":"downstairs","":"downstairs","session_id":"9320b0f0-18cb-4bf9-ad35-f17ad5dacbae"}
4952024-11-19T17:10:17.932ZDisk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
4962024-11-19T17:10:17.932ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4972024-11-19T17:10:17.932ZRead and Verify all blocks (0..50 range:true)
4982024-11-19T17:10:17.936ZWrote out file "/var/tmp/test_up-build/verify_file"
4992024-11-19T17:10:17.936ZVerify test completed
5002024-11-19T17:10:17.941ZWrote out file "/var/tmp/test_up-build/verify_file"
5012024-11-19T17:10:17.941ZCLIENT: Tests done. All submitted work has been ACK'd
5022024-11-19T17:10:17.941Z----------------------------------------------------------------
5032024-11-19T17:10:17.941Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
5042024-11-19T17:10:17.941ZStates: Active Active Active
5052024-11-19T17:10:17.941ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
5062024-11-19T17:10:17.941Z 1000 Acked Read 50 Done Done Done false
5072024-11-19T17:10:17.941Z STATES DS:0 DS:1 DS:2 TOTAL
5082024-11-19T17:10:17.941Z Sent 0 0 0 0
5092024-11-19T17:10:17.941Z Done 1 1 1 3
5102024-11-19T17:10:17.942Z Skipped 0 0 0 0
5112024-11-19T17:10:17.942Z Error 0 0 0 0
5122024-11-19T17:10:17.942ZLast Flush: 0 0 0
5132024-11-19T17:10:17.942ZDownstairs last five completed:
5142024-11-19T17:10:17.942ZUpstairs last five completed: 1000
5152024-11-19T17:10:17.942ZCLIENT: Up:0 ds:1 act:3
5162024-11-19T17:10:21.938Z----------------------------------------------------------------
5172024-11-19T17:10:21.938Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
5182024-11-19T17:10:21.938ZDownstairs last five completed: 1001 1000
5192024-11-19T17:10:21.938ZUpstairs last five completed: 1001 1000
5202024-11-19T17:10:21.938ZCLIENT: Up:0 ds:0 act:3
5212024-11-19T17:10:21.938ZCLIENT: All crucible jobs finished, exiting program
5222024-11-19T17:10:21.941Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-19T17:10:20.392743082Z","hostname":"w-01JD2P7NAF3EVMVKZHF1HVZV4F","pid":1235}
5232024-11-19T17:10:21.945ZRepair part 2 passed
5242024-11-19T17:10:21.947Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
5252024-11-19T17:10:21.962ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
5262024-11-19T17:10:21.962Z 0 000-009 32 32 32 1499 1499 1499 F F F
5272024-11-19T17:10:21.965Z 1 010-019 32 32 32 1499 1499 1499 F F F
5282024-11-19T17:10:21.965Z 2 020-029 32 32 32 1499 1499 1499 F F F
5292024-11-19T17:10:21.965Z 3 030-039 32 32 32 1499 1499 1499 F F F
5302024-11-19T17:10:21.965Z 4 040-049 32 32 32 1499 1499 1499 F F F
5312024-11-19T17:10:21.965ZMax gen: 32, Max flush: 1499
5322024-11-19T17:10:21.968Zdump test passed
5332024-11-19T17:10:21.968Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
5342024-11-19T17:10:21.982Z Extent 1
5352024-11-19T17:10:21.982ZGEN 32 32 32
5362024-11-19T17:10:21.982ZFLUSH_ID 1499 1499 1499
5372024-11-19T17:10:21.984ZDIRTY
5382024-11-19T17:10:21.984Z
5392024-11-19T17:10:21.984ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
5402024-11-19T17:10:21.986Z010 A A A A A A
5412024-11-19T17:10:21.986Z011 A A A A A A
5422024-11-19T17:10:21.988Z012 A A A A A A
5432024-11-19T17:10:21.988Z013 A A A A A A
5442024-11-19T17:10:21.992Z014 A A A A A A
5452024-11-19T17:10:21.992Z015 A A A A A A
5462024-11-19T17:10:21.994Z016 A A A A A A
5472024-11-19T17:10:21.994Z017 A A A A A A
5482024-11-19T17:10:21.994Z018 A A A A A A
5492024-11-19T17:10:21.996Z019 A A A A A A
5502024-11-19T17:10:21.998Zdump extent test passed
5512024-11-19T17:10:21.998Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
5522024-11-19T17:10:22.014ZExtent 2 Block in extent 0 Actual block 20
5532024-11-19T17:10:22.014Z
5542024-11-19T17:10:22.016Z DATA SHA256 VER
5552024-11-19T17:10:22.016Z------ ---------------------------------------------------------------- ---
5562024-11-19T17:10:22.016Z 0 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
5572024-11-19T17:10:22.016Z 1 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
5582024-11-19T17:10:22.019Z 2 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
5592024-11-19T17:10:22.019Z
5602024-11-19T17:10:22.019ZNONCES 0 1 2 DIFF
5612024-11-19T17:10:22.019Z------ ------------------------ ------------------------ ------------------------ -----
5622024-11-19T17:10:22.020Z
5632024-11-19T17:10:22.020Z TAGS 0 1 2 DIFF
5642024-11-19T17:10:22.020Z------ -------------------------------- -------------------------------- -------------------------------- -----
5652024-11-19T17:10:22.020Z
5662024-11-19T17:10:22.020ZHASHES 0 1 2 DIFF
5672024-11-19T17:10:22.020Z------ ---------------- ---------------- ---------------- -----
5682024-11-19T17:10:22.020Z 0 42916ab6fa364c70 42916ab6fa364c70 42916ab6fa364c70
5692024-11-19T17:10:22.020Z
5702024-11-19T17:10:22.022Zdump block test passed
5712024-11-19T17:10:22.022ZInitial upstairs tests have completed, stopping all downstairs
5722024-11-19T17:10:27.083ZCreating 4 larger downstairs regions
5732024-11-19T17:10:27.193ZStarting 4 downstairs
5742024-11-19T17:10:32.201Zdsc restarted at PID: 1248
5752024-11-19T17:10:32.424ZNow do the replace-reconcile test
5762024-11-19T17:10:32.477ZUsing 8840 for the replacement port
5772024-11-19T17:13:05.907ZCompleted test: replace-reconcile
5782024-11-19T17:13:05.910ZNow do the replace-before-active test
5792024-11-19T17:15:50.499ZCompleted test: replace-before-active
5802024-11-19T17:15:50.502ZAll tests have completed, stopping all downstairs
5812024-11-19T17:15:50.556Z
5822024-11-19T17:15:50.556ZAll Tests have passed
5832024-11-19T17:15:50.559Z7:49 Test duration
5842024-11-19T17:15:50.559Z
5852024-11-19T17:15:50.559Zreal 7:49.886098942
5862024-11-19T17:15:50.559Zuser 1:03.882456109
5872024-11-19T17:15:50.559Zsys 10.045076635
5882024-11-19T17:15:50.559Ztrap 0.080694762
5892024-11-19T17:15:50.559Ztflt 0.013411402
5902024-11-19T17:15:50.559Zdflt 0.049876607
5912024-11-19T17:15:50.559Zkflt 0.001159522
5922024-11-19T17:15:50.559Zlock 5:42:11.797376625
5932024-11-19T17:15:50.559Zslp 24:29.951890839
5942024-11-19T17:15:50.560Zlat 7.685034222
5952024-11-19T17:15:50.560Zstop 0.130650824
5962024-11-19T17:15:50.560Z+ echo 'test-up-unencrypted ends'
5972024-11-19T17:15:50.560Ztest-up-unencrypted ends
5982024-11-19T17:15:55.563Zprocess exited: duration 478097 ms, exit code 0
5992024-11-19T17:15:55.563Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
6002024-11-19T17:16:55.616Zexec warning: : stdout descriptor may be held open by a background process; giving up!
6012024-11-19T17:16:55.616Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
6022024-11-19T17:16:55.626Zfound 14 output files
6032024-11-19T17:16:55.626Zuploading: /tmp/test_up-build/dsc-out.txt (37453 bytes)
6042024-11-19T17:16:56.637Zuploaded: /tmp/test_up-build/dsc-out.txt
6052024-11-19T17:16:56.637Zuploading: /tmp/test_up-build/test_up_out.txt (11052272 bytes)
6062024-11-19T17:16:57.776Zuploaded: /tmp/test_up-build/test_up_out.txt
6072024-11-19T17:16:57.776Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (2502 bytes)
6082024-11-19T17:16:58.785Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
6092024-11-19T17:16:58.788Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (144658 bytes)
6102024-11-19T17:16:59.797Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
6112024-11-19T17:16:59.797Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (251085 bytes)
6122024-11-19T17:17:00.811Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
6132024-11-19T17:17:00.813Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (139428 bytes)
6142024-11-19T17:17:01.824Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
6152024-11-19T17:17:01.824Zuploading: /tmp/debug/df.txt (1270 bytes)
6162024-11-19T17:17:02.831Zuploaded: /tmp/debug/df.txt
6172024-11-19T17:17:02.833Zuploading: /tmp/debug/dtrace.txt (389771 bytes)
6182024-11-19T17:17:02.850Zupload warning: file "/tmp/debug/dtrace.txt" changed size mid upload: 389771 -> 474038
6192024-11-19T17:17:03.856Zuploaded: /tmp/debug/dtrace.txt
6202024-11-19T17:17:03.856Zuploading: /tmp/debug/iostat.txt (161044 bytes)
6212024-11-19T17:17:03.871Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 161044 -> 163753
6222024-11-19T17:17:04.878Zuploaded: /tmp/debug/iostat.txt
6232024-11-19T17:17:04.878Zuploading: /tmp/debug/mpstat.txt (390558 bytes)
6242024-11-19T17:17:04.902Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 390558 -> 397858
6252024-11-19T17:17:05.909Zuploaded: /tmp/debug/mpstat.txt
6262024-11-19T17:17:05.909Zuploading: /tmp/debug/paging.txt (70592 bytes)
6272024-11-19T17:17:05.921Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 70592 -> 72026
6282024-11-19T17:17:06.927Zuploaded: /tmp/debug/paging.txt
6292024-11-19T17:17:06.927Zuploading: /tmp/debug/prstat.txt (734236 bytes)
6302024-11-19T17:17:06.969Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 734236 -> 748471
6312024-11-19T17:17:07.976Zuploaded: /tmp/debug/prstat.txt
6322024-11-19T17:17:07.978Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
6332024-11-19T17:17:08.986Zuploaded: /tmp/debug/psrinfo.txt
6342024-11-19T17:17:08.986Zuploading: /tmp/debug/upstairs-info.txt (69160 bytes)
6352024-11-19T17:17:09.999Zuploaded: /tmp/debug/upstairs-info.txt