01JC3E2B7CWA50W82GMY1GF4GH: test-up-unencrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01JC3E2M86G2B2RV52P2X2AWMS

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-11-07T14:01:38.587Zjob dependencies complete; ready to run (waiting for 14 m 8 s)
22024-11-07T14:02:58.882Zjob assigned to worker 01JC3EWSXNYZYEKV2HKQ82GPHV (queued for 1 m 20 s)
32024-11-07T14:03:07.653Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-11-07T14:03:08.755Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-11-07T14:03:08.755Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-11-07T14:03:09.815Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-11-07T14:03:09.815Zdownloading input: /input/build/work/bins/crutest.gz
82024-11-07T14:03:10.917Zdownloaded input: /input/build/work/bins/crutest.gz
92024-11-07T14:03:10.918Zdownloading input: /input/build/work/bins/dsc.gz
102024-11-07T14:03:11.587Zdownloaded input: /input/build/work/bins/dsc.gz
112024-11-07T14:03:11.590Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-11-07T14:03:11.602Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-11-07T14:03:11.602Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-11-07T14:03:11.617Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-11-07T14:03:11.617Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-11-07T14:03:11.637Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-11-07T14:03:11.637Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-11-07T14:03:11.656Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-11-07T14:03:11.656Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-11-07T14:03:11.675Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-11-07T14:03:11.675Zdownloading input: /input/build/work/scripts/test_up.sh
222024-11-07T14:03:11.711Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-11-07T14:03:11.711Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-11-07T14:03:11.871Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-11-07T14:03:11.873Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-11-07T14:03:12.616Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-11-07T14:03:12.616Zstarting task 0: "setup"
282024-11-07T14:03:12.627Z++ uname -s
292024-11-07T14:03:12.629Z+ kern=SunOS
302024-11-07T14:03:12.629Z+ case "$kern" in
312024-11-07T14:03:12.629Z+ groupadd -g 12345 build
322024-11-07T14:03:12.632Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-11-07T14:03:14.644Z+ zfs create -o mountpoint=/work rpool/work
342024-11-07T14:03:16.098Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-11-07T14:03:16.101Z+ home_fs=zfs
362024-11-07T14:03:16.101Z+ [[ zfs == autofs ]]
372024-11-07T14:03:16.101Z+ mkdir -p /home/build
382024-11-07T14:03:16.103Z+ chown build:build /home/build /work
392024-11-07T14:03:16.106Z+ chmod 0700 /home/build /work
402024-11-07T14:03:16.108Zprocess exited: duration 3491 ms, exit code 0
 
412024-11-07T14:03:16.113Zstarting task 1: "authentication"
422024-11-07T14:03:16.131Zprocess exited: duration 17 ms, exit code 0
 
432024-11-07T14:03:16.136Zstarting task 2: "build"
442024-11-07T14:03:16.141Z+ banner cores
452024-11-07T14:03:16.162Z
462024-11-07T14:03:16.162Z #### #### ##### ###### ####
472024-11-07T14:03:16.162Z # # # # # # # #
482024-11-07T14:03:16.162Z # # # # # ##### ####
492024-11-07T14:03:16.162Z # # # ##### # #
502024-11-07T14:03:16.162Z # # # # # # # # #
512024-11-07T14:03:16.162Z #### #### # # ###### ####
522024-11-07T14:03:16.162Z
532024-11-07T14:03:16.165Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-11-07T14:03:16.173Z+ echo 'input bins dir contains:'
552024-11-07T14:03:16.176Z+ ls -ltr /input/build/work/bins
562024-11-07T14:03:16.176Zinput bins dir contains:
572024-11-07T14:03:16.176Ztotal 764763
582024-11-07T14:03:16.176Z-rw-r--r-- 1 root root 109603950 Nov 7 14:03 crucible-downstairs.gz
592024-11-07T14:03:16.176Z-rw-r--r-- 1 root root 105319952 Nov 7 14:03 crucible-hammer.gz
602024-11-07T14:03:16.176Z-rw-r--r-- 1 root root 109799934 Nov 7 14:03 crutest.gz
612024-11-07T14:03:16.176Z-rw-r--r-- 1 root root 66583768 Nov 7 14:03 dsc.gz
622024-11-07T14:03:16.176Z+ banner unpack
632024-11-07T14:03:16.179Z
642024-11-07T14:03:16.179Z # # # # ##### ## #### # #
652024-11-07T14:03:16.179Z # # ## # # # # # # # # #
662024-11-07T14:03:16.179Z # # # # # # # # # # ####
672024-11-07T14:03:16.179Z # # # # # ##### ###### # # #
682024-11-07T14:03:16.179Z # # # ## # # # # # # #
692024-11-07T14:03:16.179Z #### # # # # # #### # #
702024-11-07T14:03:16.179Z
712024-11-07T14:03:16.179Z+ mkdir -p /var/tmp/bins
722024-11-07T14:03:16.181Z+ for t in "$input/bins/"*.gz
732024-11-07T14:03:16.181Z++ basename /input/build/work/bins/crucible-downstairs.gz
742024-11-07T14:03:16.183Z+ b=crucible-downstairs.gz
752024-11-07T14:03:16.183Z+ b=crucible-downstairs
762024-11-07T14:03:16.183Z+ gunzip
772024-11-07T14:03:18.450Z+ chmod +x /var/tmp/bins/crucible-downstairs
782024-11-07T14:03:18.454Z+ for t in "$input/bins/"*.gz
792024-11-07T14:03:18.454Z++ basename /input/build/work/bins/crucible-hammer.gz
802024-11-07T14:03:18.456Z+ b=crucible-hammer.gz
812024-11-07T14:03:18.457Z+ b=crucible-hammer
822024-11-07T14:03:18.457Z+ gunzip
832024-11-07T14:03:20.632Z+ chmod +x /var/tmp/bins/crucible-hammer
842024-11-07T14:03:20.634Z+ for t in "$input/bins/"*.gz
852024-11-07T14:03:20.636Z++ basename /input/build/work/bins/crutest.gz
862024-11-07T14:03:20.636Z+ b=crutest.gz
872024-11-07T14:03:20.637Z+ b=crutest
882024-11-07T14:03:20.637Z+ gunzip
892024-11-07T14:03:22.882Z+ chmod +x /var/tmp/bins/crutest
902024-11-07T14:03:22.885Z+ for t in "$input/bins/"*.gz
912024-11-07T14:03:22.887Z++ basename /input/build/work/bins/dsc.gz
922024-11-07T14:03:22.888Z+ b=dsc.gz
932024-11-07T14:03:22.888Z+ b=dsc
942024-11-07T14:03:22.888Z+ gunzip
952024-11-07T14:03:24.291Z+ chmod +x /var/tmp/bins/dsc
962024-11-07T14:03:24.294Z+ export BINDIR=/var/tmp/bins
972024-11-07T14:03:24.294Z+ BINDIR=/var/tmp/bins
982024-11-07T14:03:24.294Z+ jobpid=1137
992024-11-07T14:03:24.296Z+ echo 'Setup debug logging'
1002024-11-07T14:03:24.296Z+ mkdir /tmp/debug
1012024-11-07T14:03:24.296ZSetup debug logging
1022024-11-07T14:03:24.296Z+ sleep 7200
1032024-11-07T14:03:24.299Z+ psrinfo -v
1042024-11-07T14:03:24.301Z+ df -h
1052024-11-07T14:03:24.439Z+ prstat -d d -mLc 1
1062024-11-07T14:03:24.442Z+ iostat -T d -xn 1
1072024-11-07T14:03:24.442Z+ mpstat -T d 1
1082024-11-07T14:03:24.442Z+ vmstat -T d -p 1
1092024-11-07T14:03:24.442Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1102024-11-07T14:03:24.444Z+ banner test_up_unencrypted
1112024-11-07T14:03:24.444Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1122024-11-07T14:03:24.447Z
1132024-11-07T14:03:24.447Z ##### ###### #### ##### # # ##### # # # #
1142024-11-07T14:03:24.447Z # # # # # # # # # # ## #
1152024-11-07T14:03:24.447Z # ##### #### # # # # # # # # # #
1162024-11-07T14:03:24.447Z # # # # # # ##### # # # # #
1172024-11-07T14:03:24.447Z # # # # # # # # # # # ##
1182024-11-07T14:03:24.447Z # ###### #### # ####### #### # ####### #### # #
1192024-11-07T14:03:24.447Z
1202024-11-07T14:03:24.447Z+ ptime -m bash /input/build/work/scripts/test_up.sh -N unencrypted
1212024-11-07T14:03:24.456Z/input/build/work
1222024-11-07T14:03:24.610ZTurn off color for downstairs dump
1232024-11-07T14:03:24.613Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1242024-11-07T14:03:24.613ZCreating 3 downstairs regions
1252024-11-07T14:03:25.240ZStarting 3 downstairs
1262024-11-07T14:03:25.243Zdsc started at PID: 1182
1272024-11-07T14:03:30.426ZDisable automatic restart on all downstairs
1282024-11-07T14:03:30.483Z
1292024-11-07T14:03:30.483ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1302024-11-07T14:03:30.487ZRunning test: span
1312024-11-07T14:03:34.632ZCompleted test: span
1322024-11-07T14:03:39.639ZRunning test: big
1332024-11-07T14:03:43.891ZCompleted test: big
1342024-11-07T14:03:48.898ZRunning test: dep
1352024-11-07T14:04:01.997ZCompleted test: dep
1362024-11-07T14:04:07.005ZRunning test: balloon
1372024-11-07T14:04:21.902ZCompleted test: balloon
1382024-11-07T14:04:26.904ZRunning test: deactivate
1392024-11-07T14:05:21.216ZCompleted test: deactivate
1402024-11-07T14:05:26.224ZRunning hammer
1412024-11-07T14:05:30.041ZRun repair tests
1422024-11-07T14:05:30.045Z/var/tmp/bins/crutest fill -g 31 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1432024-11-07T14:05:30.107Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.180821102Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"task":"crutest"}
1442024-11-07T14:05:30.110Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.181433134Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"task":"crutest"}
1452024-11-07T14:05:30.110Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.18320782Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"task":"crutest"}
1462024-11-07T14:05:30.110Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.183407665Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219}
1472024-11-07T14:05:30.110Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.1834467Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219}
1482024-11-07T14:05:30.110Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.183468847Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219}
1492024-11-07T14:05:30.110Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.183484085Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219}
1502024-11-07T14:05:30.110Z{"msg":"Crucible aa6efcab-f451-48cb-b5e3-850d834c33e9 has session id: 63b0a5cb-e4b4-4baa-9970-8af87147e959","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.183925379Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1512024-11-07T14:05:30.110Z{"msg":"Upstairs opts: Upstairs UUID: aa6efcab-f451-48cb-b5e3-850d834c33e9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.183976843Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1522024-11-07T14:05:30.110Z{"msg":"Crucible stats registered with UUID: aa6efcab-f451-48cb-b5e3-850d834c33e9","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.184016668Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1532024-11-07T14:05:30.144Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.218412544Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"task":"crutest"}
1542024-11-07T14:05:30.144Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.218489325Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219}
1552024-11-07T14:05:30.148Z{"msg":"aa6efcab-f451-48cb-b5e3-850d834c33e9 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.218573244Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1562024-11-07T14:05:30.148Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.218628267Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"client":"0","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1572024-11-07T14:05:30.148Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.218717276Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"client":"0","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1582024-11-07T14:05:30.148Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.218767629Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"client":"1","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1592024-11-07T14:05:30.148Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.218814104Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"client":"1","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1602024-11-07T14:05:30.148Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.218860808Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"client":"2","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1612024-11-07T14:05:30.148Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.218895883Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"client":"2","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1622024-11-07T14:05:30.148Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.218927959Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"":"io task","client":"0","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1632024-11-07T14:05:30.148Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.21907652Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"":"io task","client":"2","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1642024-11-07T14:05:30.148Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.219104547Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"":"io task","client":"1","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1652024-11-07T14:05:30.148Z{"msg":"ds_connection connected from Ok(127.0.0.1:33089)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.219147202Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"":"io task","client":"0","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1662024-11-07T14:05:30.148Z{"msg":"ds_connection connected from Ok(127.0.0.1:44126)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.219174378Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"":"io task","client":"1","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1672024-11-07T14:05:30.148Z{"msg":"ds_connection connected from Ok(127.0.0.1:46729)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.219220722Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"":"io task","client":"2","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1682024-11-07T14:05:30.148Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.219618982Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"client":"0","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1692024-11-07T14:05:30.148Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.219663106Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"client":"0","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1702024-11-07T14:05:30.148Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.219817167Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"client":"1","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1712024-11-07T14:05:30.148Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.219860132Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"client":"1","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1722024-11-07T14:05:30.148Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.219906066Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"client":"2","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1732024-11-07T14:05:30.148Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.219987765Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"client":"2","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1742024-11-07T14:05:30.148Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.220341551Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"client":"0","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1752024-11-07T14:05:30.148Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.220490862Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"client":"1","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1762024-11-07T14:05:30.148Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.22058085Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"client":"2","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1772024-11-07T14:05:30.148Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.220806082Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"client":"0","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1782024-11-07T14:05:30.148Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.220843437Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"client":"0","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1792024-11-07T14:05:30.148Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.220859105Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1802024-11-07T14:05:30.149Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.220873504Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"client":"1","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1812024-11-07T14:05:30.149Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.22089912Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"client":"1","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1822024-11-07T14:05:30.149Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.220930556Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1832024-11-07T14:05:30.149Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.220945004Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"client":"2","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1842024-11-07T14:05:30.149Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.220962182Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"client":"2","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1852024-11-07T14:05:30.149Z{"msg":"[0]R flush_numbers: [1526, 1526, 1526, 1526, 1526]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.220993808Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"":"downstairs","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1862024-11-07T14:05:30.149Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.221009966Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"":"downstairs","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1872024-11-07T14:05:30.149Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.221041862Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"":"downstairs","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1882024-11-07T14:05:30.149Z{"msg":"[1]R flush_numbers: [1526, 1526, 1526, 1526, 1526]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.22106132Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"":"downstairs","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1892024-11-07T14:05:30.149Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.221086237Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"":"downstairs","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1902024-11-07T14:05:30.152Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.221110263Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"":"downstairs","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1912024-11-07T14:05:30.153Z{"msg":"[2]R flush_numbers: [1526, 1526, 1526, 1526, 1526]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.221151788Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"":"downstairs","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1922024-11-07T14:05:30.153Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.221173606Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"":"downstairs","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1932024-11-07T14:05:30.153Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.221200652Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"":"downstairs","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1942024-11-07T14:05:30.153Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.2212151Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"":"downstairs","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1952024-11-07T14:05:30.153Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.221228559Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"":"downstairs","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1962024-11-07T14:05:30.153Z{"msg":"Next flush: 1527","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.221254155Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"":"downstairs","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1972024-11-07T14:05:30.153Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.221267754Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"":"downstairs","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1982024-11-07T14:05:30.153Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.221289581Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
1992024-11-07T14:05:30.153Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.221308978Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"client":"0","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
2002024-11-07T14:05:30.153Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
2012024-11-07T14:05:30.153Z2024-11-07T14:05:29.221329506Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"client":"1","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
2022024-11-07T14:05:30.153Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.221362842Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"client":"2","":"downstairs","session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
2032024-11-07T14:05:30.153Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.221394428Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
2042024-11-07T14:05:30.153Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.221415665Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
2052024-11-07T14:05:30.153Z{"msg":"aa6efcab-f451-48cb-b5e3-850d834c33e9 is now active with session: 63b0a5cb-e4b4-4baa-9970-8af87147e959","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.221446131Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
2062024-11-07T14:05:30.153Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.221460699Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219,"session_id":"63b0a5cb-e4b4-4baa-9970-8af87147e959"}
2072024-11-07T14:05:30.153Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.221473048Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219}
2082024-11-07T14:05:30.153ZRegion: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2092024-11-07T14:05:30.153ZFill test
2102024-11-07T14:05:30.160ZRead and Verify all blocks (0..50 range:false)
2112024-11-07T14:05:30.166ZWrote out file "/var/tmp/test_up-build/verify_file"
2122024-11-07T14:05:30.166ZCLIENT: Tests done. All submitted work has been ACK'd
2132024-11-07T14:05:30.169Z----------------------------------------------------------------
2142024-11-07T14:05:30.169Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2152024-11-07T14:05:30.169ZStates: Active Active Active
2162024-11-07T14:05:30.169ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2172024-11-07T14:05:30.169Z 1002 Acked Read 50 Done Done Done false
2182024-11-07T14:05:30.170Z STATES DS:0 DS:1 DS:2 TOTAL
2192024-11-07T14:05:30.170Z Sent 0 0 0 0
2202024-11-07T14:05:30.170Z Done 1 1 1 3
2212024-11-07T14:05:30.170Z Skipped 0 0 0 0
2222024-11-07T14:05:30.170Z Error 0 0 0 0
2232024-11-07T14:05:30.170ZLast Flush: 1001 1001 1001
2242024-11-07T14:05:30.170ZDownstairs last five completed: 1001 1000
2252024-11-07T14:05:30.170ZUpstairs last five completed: 1002 1001 1000
2262024-11-07T14:05:30.170ZCLIENT: Up:0 ds:1 act:3
2272024-11-07T14:05:34.168Z----------------------------------------------------------------
2282024-11-07T14:05:34.168Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2292024-11-07T14:05:34.169ZDownstairs last five completed: 1003 1002 1001 1000
2302024-11-07T14:05:34.169ZUpstairs last five completed: 1003 1002 1001 1000
2312024-11-07T14:05:34.169ZCLIENT: Up:0 ds:0 act:3
2322024-11-07T14:05:34.169ZCLIENT: All crucible jobs finished, exiting program
2332024-11-07T14:05:34.171Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:05:33.241952544Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1219}
2342024-11-07T14:05:34.181ZRepair setup passed
2352024-11-07T14:05:34.185ZCopy the region for /var/tmp/test_up-build/8830
2362024-11-07T14:05:34.188Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2372024-11-07T14:05:34.190Z/var/tmp/bins/crutest fill -g 32 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2382024-11-07T14:05:34.250Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.323157739Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"task":"crutest"}
2392024-11-07T14:05:34.252Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.323836753Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"task":"crutest"}
2402024-11-07T14:05:34.253Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.325728194Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"task":"crutest"}
2412024-11-07T14:05:34.253Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.325965924Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225}
2422024-11-07T14:05:34.253Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.326018768Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225}
2432024-11-07T14:05:34.253Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.326053823Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225}
2442024-11-07T14:05:34.253Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.326071201Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225}
2452024-11-07T14:05:34.256Z{"msg":"Crucible a1572b2e-ce73-4c31-939f-63f7f7aee6ca has session id: 250ebf9d-271f-4a03-8870-4524a5d4e615","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.326595515Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2462024-11-07T14:05:34.256Z{"msg":"Upstairs opts: Upstairs UUID: a1572b2e-ce73-4c31-939f-63f7f7aee6ca, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.326657927Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2472024-11-07T14:05:34.256Z{"msg":"Crucible stats registered with UUID: a1572b2e-ce73-4c31-939f-63f7f7aee6ca","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.326726008Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2482024-11-07T14:05:34.288Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.361226001Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"task":"crutest"}
2492024-11-07T14:05:34.288Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.36131384Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225}
2502024-11-07T14:05:34.288Z{"msg":"a1572b2e-ce73-4c31-939f-63f7f7aee6ca active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.361345336Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2512024-11-07T14:05:34.288Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.361376912Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"client":"0","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2522024-11-07T14:05:34.291Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.361428326Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"client":"0","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2532024-11-07T14:05:34.292Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.361510865Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"client":"1","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2542024-11-07T14:05:34.292Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.3615544Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"client":"1","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2552024-11-07T14:05:34.292Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.361600104Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"client":"2","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2562024-11-07T14:05:34.292Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.361651557Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"client":"2","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2572024-11-07T14:05:34.292Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.361682984Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"":"io task","client":"2","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2582024-11-07T14:05:34.292Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.36186794Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"":"io task","client":"0","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2592024-11-07T14:05:34.292Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.361973507Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"":"io task","client":"1","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2602024-11-07T14:05:34.292Z{"msg":"ds_connection connected from Ok(127.0.0.1:60879)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.362020121Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"":"io task","client":"0","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2612024-11-07T14:05:34.292Z{"msg":"ds_connection connected from Ok(127.0.0.1:35090)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.362045548Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"":"io task","client":"1","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2622024-11-07T14:05:34.292Z{"msg":"ds_connection connected from Ok(127.0.0.1:57680)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.362070685Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"":"io task","client":"2","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2632024-11-07T14:05:34.292Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.362575121Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"client":"1","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2642024-11-07T14:05:34.292Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.362614356Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"client":"1","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2652024-11-07T14:05:34.292Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.362631994Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"client":"2","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2662024-11-07T14:05:34.292Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.362691876Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"client":"2","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2672024-11-07T14:05:34.292Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.362748379Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"client":"0","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2682024-11-07T14:05:34.292Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.362784845Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"client":"0","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2692024-11-07T14:05:34.292Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.363282302Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"client":"1","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2702024-11-07T14:05:34.292Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.363433332Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"client":"0","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2712024-11-07T14:05:34.292Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.363467098Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"client":"2","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2722024-11-07T14:05:34.293Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.363684811Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"client":"1","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2732024-11-07T14:05:34.293Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.363736044Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"client":"1","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2742024-11-07T14:05:34.293Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.36376621Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2752024-11-07T14:05:34.293Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.363866868Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"client":"0","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2762024-11-07T14:05:34.293Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.363908243Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"client":"0","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2772024-11-07T14:05:34.293Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.363936199Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2782024-11-07T14:05:34.293Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.363960396Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"client":"2","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2792024-11-07T14:05:34.293Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.363994472Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"client":"2","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2802024-11-07T14:05:34.293Z{"msg":"[0]R flush_numbers: [1527, 1527, 1527, 1527, 1527]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.364021068Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"":"downstairs","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2812024-11-07T14:05:34.293Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.364057884Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"":"downstairs","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2822024-11-07T14:05:34.293Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.36408509Z",Wait for a query_work_queue command to finish before sending IO
2832024-11-07T14:05:34.293Z"hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"":"downstairs","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2842024-11-07T14:05:34.293Z{"msg":"[1]R flush_numbers: [1527, 1527, 1527, 1527, 1527]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.364131644Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"":"downstairs","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2852024-11-07T14:05:34.293Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.36416816Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"":"downstairs","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2862024-11-07T14:05:34.293Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.364194256Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"":"downstairs","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2872024-11-07T14:05:34.293Z{"msg":"[2]R flush_numbers: [1527, 1527, 1527, 1527, 1527]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.364227122Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"":"downstairs","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2882024-11-07T14:05:34.293Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.36424998Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"":"downstairs","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2892024-11-07T14:05:34.293Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.364281756Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"":"downstairs","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2902024-11-07T14:05:34.293Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.364302313Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"":"downstairs","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2912024-11-07T14:05:34.293Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.364333589Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"":"downstairs","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2922024-11-07T14:05:34.293Z{"msg":"Next flush: 1528","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.364353546Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"":"downstairs","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2932024-11-07T14:05:34.293Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.364384192Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"":"downstairs","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2942024-11-07T14:05:34.293Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible"Region: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2952024-11-07T14:05:34.293Z,"level":30,"time":"2024-11-07T14:05:33.364410069Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2962024-11-07T14:05:34.293Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.364433116Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"client":"0","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2972024-11-07T14:05:34.293Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.364459813Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"client":"1","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2982024-11-07T14:05:34.293Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.364476881Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"client":"2","":"downstairs","session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
2992024-11-07T14:05:34.293Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.364509837Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
3002024-11-07T14:05:34.293Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.364527714Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
3012024-11-07T14:05:34.293Z{"msg":"a1572b2e-ce73-4c31-939f-63f7f7aee6ca is now active with session: 250ebf9d-271f-4a03-8870-4524a5d4e615","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.36455753Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
3022024-11-07T14:05:34.297Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.364578018Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225,"session_id":"250ebf9d-271f-4a03-8870-4524a5d4e615"}
3032024-11-07T14:05:34.297Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.364607524Z","hostname":"Loading write count information from file "/var/tmp/test_up-build/verify_file"
3042024-11-07T14:05:34.297Zw-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225}
3052024-11-07T14:05:34.297ZFill test
3062024-11-07T14:05:34.305ZRead and Verify all blocks (0..50 range:false)
3072024-11-07T14:05:34.311ZWrote out file "/var/tmp/test_up-build/verify_file"
3082024-11-07T14:05:34.311ZCLIENT: Tests done. All submitted work has been ACK'd
3092024-11-07T14:05:34.314Z----------------------------------------------------------------
3102024-11-07T14:05:34.314Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
3112024-11-07T14:05:34.314ZStates: Active Active Active
3122024-11-07T14:05:34.314ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
3132024-11-07T14:05:34.314Z 1002 Acked Read 50 Done Done Done false
3142024-11-07T14:05:34.314Z STATES DS:0 DS:1 DS:2 TOTAL
3152024-11-07T14:05:34.314Z Sent 0 0 0 0
3162024-11-07T14:05:34.314Z Done 1 1 1 3
3172024-11-07T14:05:34.314Z Skipped 0 0 0 0
3182024-11-07T14:05:34.314Z Error 0 0 0 0
3192024-11-07T14:05:34.314ZLast Flush: 1001 1001 1001
3202024-11-07T14:05:34.314ZDownstairs last five completed: 1001 1000
3212024-11-07T14:05:34.314ZUpstairs last five completed: 1002 1001 1000
3222024-11-07T14:05:34.314ZCLIENT: Up:0 ds:1 act:3
3232024-11-07T14:05:38.313Z----------------------------------------------------------------
3242024-11-07T14:05:38.313Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
3252024-11-07T14:05:38.313ZDownstairs last five completed: 1003 1002 1001 1000
3262024-11-07T14:05:38.314ZUpstairs last five completed: 1003 1002 1001 1000
3272024-11-07T14:05:38.314ZCLIENT: Up:0 ds:0 act:3
3282024-11-07T14:05:38.314ZCLIENT: All crucible jobs finished, exiting program
3292024-11-07T14:05:38.316Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:05:37.386316605Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1225}
3302024-11-07T14:05:38.321ZRepair part 1 passed
3312024-11-07T14:05:38.323Z
3322024-11-07T14:05:38.323ZKill the current downstairs
3332024-11-07T14:05:38.494ZDownstairs 2 stopped
3342024-11-07T14:05:38.494Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
3352024-11-07T14:05:38.497ZNow put back the original so we have a mismatch
3362024-11-07T14:05:38.497Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
3372024-11-07T14:05:38.499ZRestart downstairs with old directory
3382024-11-07T14:05:38.557Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
3392024-11-07T14:05:38.575ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3402024-11-07T14:05:38.577Z 0 000-009 32 32 31 1528 1528 1527 F F F <---
3412024-11-07T14:05:38.578Z 1 010-019 32 32 31 1528 1528 1527 F F F <---
3422024-11-07T14:05:38.578Z 2 020-029 32 32 31 1528 1528 1527 F F F <---
3432024-11-07T14:05:38.578Z 3 030-039 32 32 31 1528 1528 1527 F F F <---
3442024-11-07T14:05:38.578Z 4 040-049 32 32 31 1528 1528 1527 F F F <---
3452024-11-07T14:05:38.578ZMax gen: 32, Max flush: 1528
3462024-11-07T14:05:38.578ZError: Difference in extent metadata found!
3472024-11-07T14:05:38.581Zdump test found error as expected
3482024-11-07T14:05:38.581Z
3492024-11-07T14:05:38.581Z
3502024-11-07T14:05:38.581Z/var/tmp/bins/crutest verify --range -g 33 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
3512024-11-07T14:05:38.641Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.713240899Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"task":"crutest"}
3522024-11-07T14:05:38.643Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.713806088Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"task":"crutest"}
3532024-11-07T14:05:38.643Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.715716047Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"task":"crutest"}
3542024-11-07T14:05:38.643Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.715906323Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233}
3552024-11-07T14:05:38.643Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.715987623Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233}
3562024-11-07T14:05:38.643Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.716031737Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233}
3572024-11-07T14:05:38.644Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.716052964Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233}
3582024-11-07T14:05:38.646Z{"msg":"Crucible ef7f8fef-50af-4c43-8ceb-1bca3ee88fbf has session id: 61aeb637-79ef-4fba-a19c-fceab6e60c25","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.716444935Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3592024-11-07T14:05:38.646Z{"msg":"Upstairs opts: Upstairs UUID: ef7f8fef-50af-4c43-8ceb-1bca3ee88fbf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.716507847Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3602024-11-07T14:05:38.646Z{"msg":"Crucible stats registered with UUID: ef7f8fef-50af-4c43-8ceb-1bca3ee88fbf","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.71656044Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3612024-11-07T14:05:38.678Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.751057274Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"task":"crutest"}
3622024-11-07T14:05:38.678Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.751111267Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233}
3632024-11-07T14:05:38.681Z{"msg":"ef7f8fef-50af-4c43-8ceb-1bca3ee88fbf active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.751210535Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3642024-11-07T14:05:38.681Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.751265688Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"client":"0","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3652024-11-07T14:05:38.681Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.751332319Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"client":"0","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3662024-11-07T14:05:38.681Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.751379433Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"client":"1","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3672024-11-07T14:05:38.681Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.751413379Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"client":"1","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3682024-11-07T14:05:38.681Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.751445225Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"client":"2","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3692024-11-07T14:05:38.681Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.75148597Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"client":"2","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3702024-11-07T14:05:38.681Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.751522525Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"":"io task","client":"2","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3712024-11-07T14:05:38.681Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.751585607Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"":"io task","client":"0","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3722024-11-07T14:05:38.681Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.751638421Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"":"io task","client":"1","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3732024-11-07T14:05:38.681Z{"msg":"ds_connection connected from Ok(127.0.0.1:39408)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.751673916Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"":"io task","client":"0","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3742024-11-07T14:05:38.681Z{"msg":"ds_connection connected from Ok(127.0.0.1:43600)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.75172307Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"":"io task","client":"1","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3752024-11-07T14:05:38.681Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:05:37.751996036Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"":"io task","client":"2","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3762024-11-07T14:05:38.681Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:05:37.752056008Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"":"io task","client":"2","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3772024-11-07T14:05:38.681Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:05:37.752087664Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3782024-11-07T14:05:38.682Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.752181512Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"":"io task","client":"2","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3792024-11-07T14:05:38.682Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.752497372Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"client":"1","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3802024-11-07T14:05:38.682Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.752532288Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"client":"1","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3812024-11-07T14:05:38.682Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.752548236Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"client":"0","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3822024-11-07T14:05:38.682Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.752587791Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"client":"0","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3832024-11-07T14:05:38.682Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.753183275Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"client":"0","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3842024-11-07T14:05:38.682Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.75322938Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"client":"1","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3852024-11-07T14:05:38.682Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.753567747Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"client":"1","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3862024-11-07T14:05:38.682Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.753601753Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"client":"1","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3872024-11-07T14:05:38.682Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.753617201Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3882024-11-07T14:05:38.682Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.753638308Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"client":"0","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3892024-11-07T14:05:38.682Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.753652516Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"client":"0","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3902024-11-07T14:05:38.682Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.753667054Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3912024-11-07T14:05:48.682Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.7531276Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"":"io task","client":"2","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3922024-11-07T14:05:48.686Z{"msg":"ds_connection connected from Ok(127.0.0.1:51293)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.753282571Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"":"io task","client":"2","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3932024-11-07T14:05:48.686Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.754308511Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"client":"2","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3942024-11-07T14:05:48.686Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.754350556Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"client":"2","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3952024-11-07T14:05:48.686Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.75487275Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"client":"2","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3962024-11-07T14:05:48.686Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.755246743Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"client":"2","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3972024-11-07T14:05:48.686Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.755277579Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"client":"2","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3982024-11-07T14:05:48.687Z{"msg":"[0]R flush_numbers: [1528, 1528, 1528, 1528, 1528]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.755293277Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
3992024-11-07T14:05:48.687Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.755308095Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4002024-11-07T14:05:48.687Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.755322343Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4012024-11-07T14:05:48.687Z{"msg":"[1]R flush_numbers: [1528, 1528, 1528, 1528, 1528]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.755338151Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4022024-11-07T14:05:48.687Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.755359539Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4032024-11-07T14:05:48.687Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.755373907Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4042024-11-07T14:05:48.687Z{"msg":"[2]R flush_numbers: [1527, 1527, 1527, 1527, 1527]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.755394884Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4052024-11-07T14:05:48.687Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.755408903Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4062024-11-07T14:05:48.687Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.75542341Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4072024-11-07T14:05:48.687Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.755445938Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4082024-11-07T14:05:48.687Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.755460136Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4092024-11-07T14:05:48.687Z{"msg":"Next flush: 1529","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.755481243Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4102024-11-07T14:05:48.687Z{"msg":"Extent 0 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.755495152Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4112024-11-07T14:05:48.687Z{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.755516549Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4122024-11-07T14:05:48.687Z{"msg":"extent:0 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.755531837Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4132024-11-07T14:05:48.687Z{"msg":"extent:0 flush: 1528 1528 1527 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.755546875Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4142024-11-07T14:05:48.687Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.755561683Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4152024-11-07T14:05:48.687Z{"msg":"extent:0 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.75558356Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4162024-11-07T14:05:48.687Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.755598899Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4172024-11-07T14:05:48.687Z{"msg":"find dest for source 0 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.755620686Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4182024-11-07T14:05:48.687Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.755635574Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4192024-11-07T14:05:48.687Z{"msg":"Extent 1 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.75566965Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4202024-11-07T14:05:48.687Z{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.755693696Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4212024-11-07T14:05:48.687Z{"msg":"extent:1 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.755709394Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4222024-11-07T14:05:48.687Z{"msg":"extent:1 flush: 1528 1528 1527 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.755731542Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4232024-11-07T14:05:48.687Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.7557466Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4242024-11-07T14:05:48.687Z{"msg":"extent:1 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.755769527Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4252024-11-07T14:05:48.687Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.755786105Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4262024-11-07T14:05:48.687Z{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.755808702Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4272024-11-07T14:05:48.687Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.75582723Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4282024-11-07T14:05:48.687Z{"msg":"Extent 2 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.755857176Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4292024-11-07T14:05:48.687Z{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.75589947Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4302024-11-07T14:05:48.687Z{"msg":"extent:2 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.755927557Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4312024-11-07T14:05:48.687Z{"msg":"extent:2 flush: 1528 1528 1527 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.755962093Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4322024-11-07T14:05:48.687Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.75598375Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4332024-11-07T14:05:48.687Z{"msg":"extent:2 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.756017555Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4342024-11-07T14:05:48.687Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.756037843Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4352024-11-07T14:05:48.687Z{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.756072599Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4362024-11-07T14:05:48.688Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.756099625Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4372024-11-07T14:05:48.688Z{"msg":"Extent 3 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.756119313Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4382024-11-07T14:05:48.691Z{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.756157658Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4392024-11-07T14:05:48.691Z{"msg":"extent:3 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.756191314Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4402024-11-07T14:05:48.691Z{"msg":"extent:3 flush: 1528 1528 1527 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.756224089Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4412024-11-07T14:05:48.691Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.756274083Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4422024-11-07T14:05:48.691Z{"msg":"extent:3 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.756329936Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4432024-11-07T14:05:48.691Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.756384499Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4442024-11-07T14:05:48.691Z{"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.756410126Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4452024-11-07T14:05:48.691Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.756434993Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4462024-11-07T14:05:48.691Z{"msg":"Extent 4 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.756451161Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4472024-11-07T14:05:48.691Z{"msg":"First source client ID for extent 4","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.756465109Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4482024-11-07T14:05:48.691Z{"msg":"extent:4 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.756487436Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4492024-11-07T14:05:48.691Z{"msg":"extent:4 flush: 1528 1528 1527 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.756503504Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4502024-11-07T14:05:48.692Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.756518952Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4512024-11-07T14:05:48.692Z{"msg":"extent:4 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.75653377Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4522024-11-07T14:05:48.692Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.756556358Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4532024-11-07T14:05:48.692Z{"msg":"find dest for source 0 for extent at index 4","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.756571676Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4542024-11-07T14:05:48.692Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.756586884Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4552024-11-07T14:05:48.692Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.756602162Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"client":"0","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4562024-11-07T14:05:48.692Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.75661611Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"client":"1","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4572024-11-07T14:05:48.692Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.756630938Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"client":"2","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4582024-11-07T14:05:48.692Z{"msg":"starting reconciliation 3aaff96d-d204-4b70-87d2-133c099260f6: found 5 extents that need repair","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.756655265Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4592024-11-07T14:05:48.692Z{"msg":"Full repair list: {ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.756673803Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4602024-11-07T14:05:48.692Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 1529, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 1529, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 1529, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 1529, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 1529, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.756746694Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4612024-11-07T14:05:48.692Z{"msg":"no Nexus client from DNS, aborting start notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:05:47.756763072Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"reconcile":"3aaff96d-d204-4b70-87d2-133c099260f6","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4622024-11-07T14:05:48.692Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:05:47.75677841Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"reconcile":"3aaff96d-d204-4b70-87d2-133c099260f6","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4632024-11-07T14:05:48.692Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:05:47.756794017Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"reconcile":"3aaff96d-d204-4b70-87d2-133c099260f6","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4642024-11-07T14:05:48.692Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.756816255Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"client":"2","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4652024-11-07T14:05:48.692Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:05:47.756830423Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"reconcile":"3aaff96d-d204-4b70-87d2-133c099260f6","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4662024-11-07T14:05:48.699Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:05:47.770148531Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"reconcile":"3aaff96d-d204-4b70-87d2-133c099260f6","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4672024-11-07T14:05:48.701Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:05:47.770645808Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"reconcile":"3aaff96d-d204-4b70-87d2-133c099260f6","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4682024-11-07T14:05:48.701Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:05:47.77094573Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"reconcile":"3aaff96d-d204-4b70-87d2-133c099260f6","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4692024-11-07T14:05:48.701Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.771213936Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"client":"2","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4702024-11-07T14:05:48.701Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:05:47.771270829Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"reconcile":"3aaff96d-d204-4b70-87d2-133c099260f6","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4712024-11-07T14:05:48.712Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:05:47.782971741Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"reconcile":"3aaff96d-d204-4b70-87d2-133c099260f6","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4722024-11-07T14:05:48.714Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:05:47.783468018Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"reconcile":"3aaff96d-d204-4b70-87d2-133c099260f6","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4732024-11-07T14:05:48.714Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:05:47.783799496Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"reconcile":"3aaff96d-d204-4b70-87d2-133c099260f6","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4742024-11-07T14:05:48.714Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.784096819Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"client":"2","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4752024-11-07T14:05:48.714Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:05:47.784131444Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"reconcile":"3aaff96d-d204-4b70-87d2-133c099260f6","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4762024-11-07T14:05:48.723Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:05:47.794311379Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"reconcile":"3aaff96d-d204-4b70-87d2-133c099260f6","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4772024-11-07T14:05:48.725Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:05:47.794808696Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"reconcile":"3aaff96d-d204-4b70-87d2-133c099260f6","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4782024-11-07T14:05:48.725Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:05:47.795101729Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"reconcile":"3aaff96d-d204-4b70-87d2-133c099260f6","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4792024-11-07T14:05:48.726Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.795421059Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"client":"2","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4802024-11-07T14:05:48.726Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:05:47.795475512Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"reconcile":"3aaff96d-d204-4b70-87d2-133c099260f6","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4812024-11-07T14:05:48.733Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:05:47.804804854Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"reconcile":"3aaff96d-d204-4b70-87d2-133c099260f6","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4822024-11-07T14:05:48.736Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:05:47.805326178Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"reconcile":"3aaff96d-d204-4b70-87d2-133c099260f6","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4832024-11-07T14:05:48.736Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:05:47.805660785Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"reconcile":"3aaff96d-d204-4b70-87d2-133c099260f6","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4842024-11-07T14:05:48.736Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.80594504Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"client":"2","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4852024-11-07T14:05:48.736Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:05:47.806000773Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"reconcile":"3aaff96d-d204-4b70-87d2-133c099260f6","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4862024-11-07T14:05:48.744Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:05:47.815513461Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"reconcile":"3aaff96d-d204-4b70-87d2-133c099260f6","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4872024-11-07T14:05:48.747Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.815932459Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4882024-11-07T14:05:48.747Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.815975043Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"client":"0","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4892024-11-07T14:05:48.747Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.81599653Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"client":"1","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4902024-11-07T14:05:48.747Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.816015478Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"client":"2","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4912024-11-07T14:05:48.747Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.816041915Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4922024-11-07T14:05:48.747Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.816064492Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4932024-11-07T14:05:48.747ZWait for a query_work_queue command to finish before sending IO
4942024-11-07T14:05:48.747Z{"msg":"ef7f8fef-50af-4c43-8ceb-1bca3ee88fbf is now active with session: 61aeb637-79ef-4fba-a19c-fceab6e60c25","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.816108056Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4952024-11-07T14:05:48.747Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:47.81615356Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233}
4962024-11-07T14:05:48.747Z{"msg":"no Nexus client from DNS, aborting finish notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:05:47.816198975Z","hostname":"w-01JC3EWSXNYZYEKV2HKQ82GPHV","pid":1233,"reconcile":"3aaff96d-d204-4b70-87d2-133c099260f6","":"downstairs","":"downstairs","session_id":"61aeb637-79ef-4fba-a19c-fceab6e60c25"}
4972024-11-07T14:05:48.747ZRegion: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
4982024-11-07T14:05:48.747ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4992024-11-07T14:05:48.747ZRead and Verify all blocks (0..50 range:true)
5002024-11-07T14:05:48.752ZWrote out file "/var/tmp/test_up-build/verify_file"
5012024-11-07T14:05:48.752ZVerify test completed
5022024-11-07T14:05:48.755ZWrote out file "/var/tmp/test_up-build/verify_file"
5032024-11-07T14:05:48.755ZCLIENT: Tests done. All submitted work has been ACK'd
5042024-11-07T14:05:48.755Z----------------------------------------------------------------
5052024-11-07T14:05:48.755Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
5062024-11-07T14:05:48.755ZStates: Active Active Active
5072024-11-07T14:05:48.755ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
5082024-11-07T14:05:48.755Z 1000 Acked Read 50 Done Done Done false
5092024-11-07T14:05:48.755Z STATES DS:0 DS:1 DS:2 TOTAL
5102024-11-07T14:05:48.755Z Sent 0 0 0 0
5112024-11-07T14:05:48.755Z Done 1 1 1 3
5122024-11-07T14:05:48.755Z Skipped 0 0 0 0
5132024-11-07T14:05:48.755Z Error 0 0 0 0
5142024-11-07T14:05:48.755ZLast Flush: 0 0 0
5152024-11-07T14:05:48.755ZDownstairs last five completed:
5162024-11-07T14:05:48.755ZUpstairs last five completed: 1000
5172024-11-07T14:05:48.755ZCLIENT: Up:0 ds:1 act:3
5182024-11-07T14:05:52.755Z----------------------------------------------------------------
5192024-11-07T14:05:52.755Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
5202024-11-07T14:05:52.755ZDownstairs last five completed: 1001 1000
5212024-11-07T14:05:52.755ZUpstairs last five completed: 1001 1000
5222024-11-07T14:05:52.755ZCLIENT: Up:0 ds:0 act:3
5232024-11-07T14:05:52.755ZCLIENT: All crucible jobs finished, exiting program
5242024-11-07T14:05:52.762ZRepair part 2 passed
5252024-11-07T14:05:52.765Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
5262024-11-07T14:05:52.780ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
5272024-11-07T14:05:52.780Z 0 000-009 32 32 32 1528 1528 1528 F F F
5282024-11-07T14:05:52.782Z 1 010-019 32 32 32 1528 1528 1528 F F F
5292024-11-07T14:05:52.782Z 2 020-029 32 32 32 1528 1528 1528 F F F
5302024-11-07T14:05:52.782Z 3 030-039 32 32 32 1528 1528 1528 F F F
5312024-11-07T14:05:52.782Z 4 040-049 32 32 32 1528 1528 1528 F F F
5322024-11-07T14:05:52.782ZMax gen: 32, Max flush: 1528
5332024-11-07T14:05:52.785Zdump test passed
5342024-11-07T14:05:52.785Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
5352024-11-07T14:05:52.802Z Extent 1
5362024-11-07T14:05:52.805ZGEN 32 32 32
5372024-11-07T14:05:52.805ZFLUSH_ID 1528 1528 1528
5382024-11-07T14:05:52.805ZDIRTY
5392024-11-07T14:05:52.805Z
5402024-11-07T14:05:52.805ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
5412024-11-07T14:05:52.805Z010 A A A A A A
5422024-11-07T14:05:52.805Z011 A A A A A A
5432024-11-07T14:05:52.808Z012 A A A A A A
5442024-11-07T14:05:52.808Z013 A A A A A A
5452024-11-07T14:05:52.808Z014 A A A A A A
5462024-11-07T14:05:52.817Z015 A A A A A A
5472024-11-07T14:05:52.817Z016 A A A A A A
5482024-11-07T14:05:52.817Z017 A A A A A A
5492024-11-07T14:05:52.817Z018 A A A A A A
5502024-11-07T14:05:52.817Z019 A A A A A A
5512024-11-07T14:05:52.820Zdump extent test passed
5522024-11-07T14:05:52.821Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
5532024-11-07T14:05:52.835ZExtent 2 Block in extent 0 Actual block 20
5542024-11-07T14:05:52.835Z
5552024-11-07T14:05:52.837Z DATA SHA256 VER
5562024-11-07T14:05:52.837Z------ ---------------------------------------------------------------- ---
5572024-11-07T14:05:52.837Z 0 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
5582024-11-07T14:05:52.837Z 1 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
5592024-11-07T14:05:52.840Z 2 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
5602024-11-07T14:05:52.840Z
5612024-11-07T14:05:52.840ZNONCES 0 1 2 DIFF
5622024-11-07T14:05:52.840Z------ ------------------------ ------------------------ ------------------------ -----
5632024-11-07T14:05:52.840Z
5642024-11-07T14:05:52.840Z TAGS 0 1 2 DIFF
5652024-11-07T14:05:52.840Z------ -------------------------------- -------------------------------- -------------------------------- -----
5662024-11-07T14:05:52.840Z
5672024-11-07T14:05:52.840ZHASHES 0 1 2 DIFF
5682024-11-07T14:05:52.840Z------ ---------------- ---------------- ---------------- -----
5692024-11-07T14:05:52.840Z 0 42916ab6fa364c70 42916ab6fa364c70 42916ab6fa364c70
5702024-11-07T14:05:52.840Z
5712024-11-07T14:05:52.842Zdump block test passed
5722024-11-07T14:05:52.842ZInitial upstairs tests have completed, stopping all downstairs
5732024-11-07T14:05:57.907ZCreating 4 larger downstairs regions
5742024-11-07T14:05:58.513ZStarting 4 downstairs
5752024-11-07T14:06:03.521Zdsc restarted at PID: 1246
5762024-11-07T14:06:03.757ZNow do the replace-reconcile test
5772024-11-07T14:06:03.814ZUsing 8840 for the replacement port
5782024-11-07T14:09:13.418ZCompleted test: replace-reconcile
5792024-11-07T14:09:13.420ZNow do the replace-before-active test
5802024-11-07T14:11:58.202ZCompleted test: replace-before-active
5812024-11-07T14:11:58.202ZAll tests have completed, stopping all downstairs
5822024-11-07T14:11:58.259Z
5832024-11-07T14:11:58.262ZAll Tests have passed
5842024-11-07T14:11:58.262Z8:34 Test duration
5852024-11-07T14:11:58.262Z
5862024-11-07T14:11:58.262Zreal 8:33.737149826
5872024-11-07T14:11:58.262Zuser 1:04.808600955
5882024-11-07T14:11:58.262Zsys 12.410098882
5892024-11-07T14:11:58.262Ztrap 0.151926249
5902024-11-07T14:11:58.262Ztflt 0.014212219
5912024-11-07T14:11:58.262Zdflt 0.332484052
5922024-11-07T14:11:58.262Zkflt 0.001200204
5932024-11-07T14:11:58.262Zlock 6:05:37.004355997
5942024-11-07T14:11:58.262Zslp 26:37.452469043
5952024-11-07T14:11:58.262Zlat 9.790723161
5962024-11-07T14:11:58.262Zstop 0.145044828
5972024-11-07T14:11:58.263Z+ echo 'test-up-unencrypted ends'
5982024-11-07T14:11:58.263Ztest-up-unencrypted ends
5992024-11-07T14:12:03.266Zprocess exited: duration 522049 ms, exit code 0
6002024-11-07T14:12:03.266Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
6012024-11-07T14:13:03.304Zexec warning: : stdout descriptor may be held open by a background process; giving up!
6022024-11-07T14:13:03.304Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
6032024-11-07T14:13:03.314Zfound 14 output files
6042024-11-07T14:13:03.314Zuploading: /tmp/test_up-build/dsc-out.txt (41926 bytes)
6052024-11-07T14:13:04.325Zuploaded: /tmp/test_up-build/dsc-out.txt
6062024-11-07T14:13:04.334Zuploading: /tmp/test_up-build/test_up_out.txt (10525302 bytes)
6072024-11-07T14:13:05.542Zuploaded: /tmp/test_up-build/test_up_out.txt
6082024-11-07T14:13:05.545Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (2504 bytes)
6092024-11-07T14:13:06.551Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
6102024-11-07T14:13:06.553Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (144643 bytes)
6112024-11-07T14:13:07.569Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
6122024-11-07T14:13:07.578Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (251103 bytes)
6132024-11-07T14:13:08.593Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
6142024-11-07T14:13:08.595Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (139423 bytes)
6152024-11-07T14:13:09.610Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
6162024-11-07T14:13:09.612Zuploading: /tmp/debug/df.txt (1270 bytes)
6172024-11-07T14:13:10.620Zuploaded: /tmp/debug/df.txt
6182024-11-07T14:13:10.623Zuploading: /tmp/debug/dtrace.txt (515810 bytes)
6192024-11-07T14:13:11.652Zuploaded: /tmp/debug/dtrace.txt
6202024-11-07T14:13:11.652Zuploading: /tmp/debug/iostat.txt (174379 bytes)
6212024-11-07T14:13:11.668Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 174379 -> 177088
6222024-11-07T14:13:12.673Zuploaded: /tmp/debug/iostat.txt
6232024-11-07T14:13:12.676Zuploading: /tmp/debug/mpstat.txt (422678 bytes)
6242024-11-07T14:13:12.698Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 422678 -> 429248
6252024-11-07T14:13:13.704Zuploaded: /tmp/debug/mpstat.txt
6262024-11-07T14:13:13.707Zuploading: /tmp/debug/paging.txt (76328 bytes)
6272024-11-07T14:13:13.714Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 76328 -> 77762
6282024-11-07T14:13:14.721Zuploaded: /tmp/debug/paging.txt
6292024-11-07T14:13:14.723Zuploading: /tmp/debug/prstat.txt (800569 bytes)
6302024-11-07T14:13:14.762Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 800569 -> 816107
6312024-11-07T14:13:15.768Zuploaded: /tmp/debug/prstat.txt
6322024-11-07T14:13:15.771Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
6332024-11-07T14:13:16.777Zuploaded: /tmp/debug/psrinfo.txt
6342024-11-07T14:13:16.780Zuploading: /tmp/debug/upstairs-info.txt (79158 bytes)
6352024-11-07T14:13:17.790Zuploaded: /tmp/debug/upstairs-info.txt