01HVPFE52N1MC7BPVNEMFE1HVH: test-up-encrypted

BasicConfig {
    output_rules: [
        "/tmp/test_up/*.txt",
        "/tmp/test_up/dsc/*.txt",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01HVPFEFQQYGYF83C577DXH6CE

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-04-17T17:07:18.383Zjob dependencies complete; ready to run (waiting for 17 m 8 s)
22024-04-17T17:09:29.062Zjob assigned to worker 01HVPGE0ZWY84ZKN7SCZ1H0XWF (queued for 2 m 10 s)
32024-04-17T17:09:36.274Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-04-17T17:09:49.904Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-04-17T17:09:49.904Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-04-17T17:10:01.726Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-04-17T17:10:01.726Zdownloading input: /input/build/work/bins/crutest.gz
82024-04-17T17:10:09.759Zdownloaded input: /input/build/work/bins/crutest.gz
92024-04-17T17:10:09.759Zdownloading input: /input/build/work/bins/dsc.gz
102024-04-17T17:10:17.606Zdownloaded input: /input/build/work/bins/dsc.gz
112024-04-17T17:10:17.611Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-04-17T17:10:17.653Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-04-17T17:10:17.654Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-04-17T17:10:17.669Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-04-17T17:10:17.669Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-04-17T17:10:17.688Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-04-17T17:10:17.688Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-04-17T17:10:17.709Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-04-17T17:10:17.709Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-04-17T17:10:17.743Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-04-17T17:10:17.751Zdownloading input: /input/build/work/scripts/test_up.sh
222024-04-17T17:10:17.764Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-04-17T17:10:17.781Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-04-17T17:10:17.791Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-04-17T17:10:17.798Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-04-17T17:10:35.531Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-04-17T17:10:35.531Zstarting task 0: "setup"
282024-04-17T17:10:35.612Z++ uname -s
292024-04-17T17:10:35.615Z+ kern=SunOS
302024-04-17T17:10:35.616Z+ case "$kern" in
312024-04-17T17:10:35.616Z+ groupadd -g 12345 build
322024-04-17T17:10:35.648Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-04-17T17:10:37.668Z+ zfs create -o mountpoint=/work rpool/work
342024-04-17T17:10:37.800Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-04-17T17:10:37.807Z+ home_fs=zfs
362024-04-17T17:10:37.807Z+ [[ zfs == autofs ]]
372024-04-17T17:10:37.808Z+ mkdir -p /home/build
382024-04-17T17:10:37.808Z+ chown build:build /home/build /work
392024-04-17T17:10:39.808Z+ chmod 0700 /home/build /work
402024-04-17T17:10:39.832Zprocess exited: duration 4275 ms, exit code 0
 
412024-04-17T17:10:39.852Zstarting task 1: "authentication"
422024-04-17T17:10:39.872Zprocess exited: duration 17 ms, exit code 0
 
432024-04-17T17:10:39.898Zstarting task 2: "build"
442024-04-17T17:10:39.914Z+ banner cores
452024-04-17T17:10:39.914Z
462024-04-17T17:10:39.914Z #### #### ##### ###### ####
472024-04-17T17:10:39.914Z # # # # # # # #
482024-04-17T17:10:39.914Z # # # # # ##### ####
492024-04-17T17:10:39.914Z # # # ##### # #
502024-04-17T17:10:39.914Z # # # # # # # # #
512024-04-17T17:10:39.914Z #### #### # # ###### ####
522024-04-17T17:10:39.914Z
532024-04-17T17:10:39.914Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-04-17T17:10:39.936Z+ echo 'input bins dir contains:'
552024-04-17T17:10:39.937Zinput bins dir contains:
562024-04-17T17:10:39.937Z+ ls -ltr /input/build/work/bins
572024-04-17T17:10:39.937Ztotal 659845
582024-04-17T17:10:39.937Z-rw-r--r-- 1 root root 98275455 Apr 17 17:09 crucible-downstairs.gz
592024-04-17T17:10:39.937Z-rw-r--r-- 1 root root 79853354 Apr 17 17:09 crucible-hammer.gz
602024-04-17T17:10:39.937Z-rw-r--r-- 1 root root 95763310 Apr 17 17:10 crutest.gz
612024-04-17T17:10:39.937Z-rw-r--r-- 1 root root 63730486 Apr 17 17:10 dsc.gz
622024-04-17T17:10:39.938Z+ banner unpack
632024-04-17T17:10:39.938Z
642024-04-17T17:10:39.938Z # # # # ##### ## #### # #
652024-04-17T17:10:39.938Z # # ## # # # # # # # # #
662024-04-17T17:10:39.938Z # # # # # # # # # # ####
672024-04-17T17:10:39.938Z # # # # # ##### ###### # # #
682024-04-17T17:10:39.939Z # # # ## # # # # # # #
692024-04-17T17:10:39.939Z #### # # # # # #### # #
702024-04-17T17:10:39.939Z
712024-04-17T17:10:39.939Z+ mkdir -p /var/tmp/bins
722024-04-17T17:10:39.939Z+ for t in "$input/bins/"*.gz
732024-04-17T17:10:39.939Z++ basename /input/build/work/bins/crucible-downstairs.gz
742024-04-17T17:10:39.940Z+ b=crucible-downstairs.gz
752024-04-17T17:10:39.940Z+ b=crucible-downstairs
762024-04-17T17:10:39.940Z+ gunzip
772024-04-17T17:10:41.982Z+ chmod +x /var/tmp/bins/crucible-downstairs
782024-04-17T17:10:41.996Z+ for t in "$input/bins/"*.gz
792024-04-17T17:10:41.997Z++ basename /input/build/work/bins/crucible-hammer.gz
802024-04-17T17:10:41.997Z+ b=crucible-hammer.gz
812024-04-17T17:10:41.997Z+ b=crucible-hammer
822024-04-17T17:10:41.997Z+ gunzip
832024-04-17T17:10:43.663Z+ chmod +x /var/tmp/bins/crucible-hammer
842024-04-17T17:10:43.668Z+ for t in "$input/bins/"*.gz
852024-04-17T17:10:43.668Z++ basename /input/build/work/bins/crutest.gz
862024-04-17T17:10:43.669Z+ b=crutest.gz
872024-04-17T17:10:43.669Z+ b=crutest
882024-04-17T17:10:43.669Z+ gunzip
892024-04-17T17:10:45.672Z+ chmod +x /var/tmp/bins/crutest
902024-04-17T17:10:45.676Z+ for t in "$input/bins/"*.gz
912024-04-17T17:10:45.676Z++ basename /input/build/work/bins/dsc.gz
922024-04-17T17:10:45.679Z+ b=dsc.gz
932024-04-17T17:10:45.679Z+ b=dsc
942024-04-17T17:10:45.679Z+ gunzip
952024-04-17T17:10:47.362Z+ chmod +x /var/tmp/bins/dsc
962024-04-17T17:10:47.366Z+ export BINDIR=/var/tmp/bins
972024-04-17T17:10:47.366Z+ BINDIR=/var/tmp/bins
982024-04-17T17:10:47.366Z+ banner test_up_encrypted
992024-04-17T17:10:47.372Z
1002024-04-17T17:10:47.372Z ##### ###### #### ##### # # ##### ###### # #
1012024-04-17T17:10:47.372Z # # # # # # # # # ## #
1022024-04-17T17:10:47.372Z # ##### #### # # # # # ##### # # #
1032024-04-17T17:10:47.372Z # # # # # # ##### # # # #
1042024-04-17T17:10:47.373Z # # # # # # # # # # ##
1052024-04-17T17:10:47.373Z # ###### #### # ####### #### # ####### ###### # #
1062024-04-17T17:10:47.373Z
1072024-04-17T17:10:47.373Z+ ptime -m bash /input/build/work/scripts/test_up.sh -N encrypted
1082024-04-17T17:10:47.501Z/input/build/work
1092024-04-17T17:10:47.509ZTurn off color for downstairs dump
1102024-04-17T17:10:47.824ZUpstairs using key: GAHk0Qm9zkd65cUouA2Rh9epFmhVPivSi1D/01vBwgc=
1112024-04-17T17:10:47.827Zdsc output goes to /tmp/test_up/dsc-out.txt
1122024-04-17T17:10:47.827ZCreating three downstairs regions
1132024-04-17T17:10:48.004ZStarting three downstairs
1142024-04-17T17:10:53.016ZDisable automatic restart on all downstairs
1152024-04-17T17:10:53.073Z
1162024-04-17T17:10:53.073ZBegin tests, output goes to /tmp/test_up/test_up_out.txt
1172024-04-17T17:10:53.097ZRunning test: span
1182024-04-17T17:10:58.289ZCompleted test: span
1192024-04-17T17:11:02.138ZRunning test: big
1202024-04-17T17:11:06.443ZCompleted test: big
1212024-04-17T17:11:11.451ZRunning test: dep
1222024-04-17T17:11:54.494ZCompleted test: dep
1232024-04-17T17:11:59.503ZRunning test: balloon
1242024-04-17T17:12:41.159ZCompleted test: balloon
1252024-04-17T17:12:46.166ZRunning test: deactivate
1262024-04-17T17:13:40.796ZCompleted test: deactivate
1272024-04-17T17:13:45.806ZRunning hammer
1282024-04-17T17:14:02.266ZRun repair tests
1292024-04-17T17:14:02.274Z/var/tmp/bins/crutest fill -g 31 -q --key GAHk0Qm9zkd65cUouA2Rh9epFmhVPivSi1D/01vBwgc= -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --verify-out /var/tmp/test_up/verify_file
1302024-04-17T17:14:02.281Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.77083469Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206}
1312024-04-17T17:14:02.281Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.771140451Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206}
1322024-04-17T17:14:02.289Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.771170717Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206}
1332024-04-17T17:14:02.289Z{"msg":"Crucible ead2ccf7-0d55-4a5f-931b-5837bde4d520 has session id: 97bcd462-ff38-40bd-9ab5-6ce1e1b72990","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.771205901Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1342024-04-17T17:14:02.289Z{"msg":"Upstairs opts: Upstairs UUID: ead2ccf7-0d55-4a5f-931b-5837bde4d520, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.771258733Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1352024-04-17T17:14:02.289Z{"msg":"Crucible stats registered with UUID: ead2ccf7-0d55-4a5f-931b-5837bde4d520","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.771293777Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1362024-04-17T17:14:02.289ZCrucible runtime is spawned
1372024-04-17T17:14:02.289Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.771412168Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206}
1382024-04-17T17:14:02.289Z{"msg":"ead2ccf7-0d55-4a5f-931b-5837bde4d520 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.771446383Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1392024-04-17T17:14:02.289Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.771462771Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"client":"0","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1402024-04-17T17:14:02.289Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.771481668Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"client":"0","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1412024-04-17T17:14:02.289Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.771498735Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"client":"1","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1422024-04-17T17:14:02.289Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.771524451Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"client":"1","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1432024-04-17T17:14:02.289Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.771541508Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"client":"2","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1442024-04-17T17:14:02.289Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.771562525Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"client":"2","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1452024-04-17T17:14:02.289Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.771601578Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1462024-04-17T17:14:02.289Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.771687725Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1472024-04-17T17:14:02.289Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.771764633Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1482024-04-17T17:14:02.289Z{"msg":"ds_connection connected from Ok(127.0.0.1:65332)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.771824683Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1492024-04-17T17:14:02.289Z{"msg":"ds_connection connected from Ok(127.0.0.1:42384)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.772048388Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1502024-04-17T17:14:02.290Z{"msg":"ds_connection connected from Ok(127.0.0.1:57807)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.77210235Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1512024-04-17T17:14:02.290Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.772520733Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"client":"0","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1522024-04-17T17:14:02.290Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.772580044Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"client":"0","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1532024-04-17T17:14:02.290Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.772628806Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"client":"1","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1542024-04-17T17:14:02.290Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.772659571Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"client":"1","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1552024-04-17T17:14:02.290Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.772699885Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"client":"2","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1562024-04-17T17:14:02.290Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.772755076Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"client":"2","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1572024-04-17T17:14:02.290Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.773121089Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"client":"2","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1582024-04-17T17:14:02.290Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.773157963Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"client":"1","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1592024-04-17T17:14:02.290Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.773226602Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"client":"0","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1602024-04-17T17:14:02.290Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.773543812Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"client":"2","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1612024-04-17T17:14:02.290Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.773580076Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"client":"2","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1622024-04-17T17:14:02.290Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.773595843Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1632024-04-17T17:14:02.290Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.773608832Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"client":"1","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1642024-04-17T17:14:02.290Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.773630038Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"client":"1","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1652024-04-17T17:14:02.290Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.773643866Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1662024-04-17T17:14:02.290Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.773656664Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"client":"0","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1672024-04-17T17:14:02.290Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.773686009Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"client":"0","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1682024-04-17T17:14:02.290Z{"msg":"Wait for a query_work_queue command to finish before sending IO
1692024-04-17T17:14:02.290Z[0]R flush_numbers: [1658, 1658, 1658, 1658, 1658]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.773712695Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"":"downstairs","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1702024-04-17T17:14:02.291Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.77374632Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"":"downstairs","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1712024-04-17T17:14:02.291Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.773780145Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"":"downstairs","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1722024-04-17T17:14:02.291Z{"msg":"[1]R flush_numbers: [1658, 1658, 1658, 1658, 1658]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.773816459Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"":"downstairs","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1732024-04-17T17:14:02.291Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.773832406Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"":"downstairs","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1742024-04-17T17:14:02.291Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.773857852Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"":"downstairs","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1752024-04-17T17:14:02.291Z{"msg":"[2]R flush_numbers: [1658, 1658, 1658, 1658, 1658]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.773894136Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"":"downstairs","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1762024-04-17T17:14:02.291Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.773925322Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"":"downstairs","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1772024-04-17T17:14:02.291Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.773946328Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"":"downstairs","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1782024-04-17T17:14:02.291Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.773980853Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"":"downstairs","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1792024-04-17T17:14:02.291Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.774005429Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"":"downstairs","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1802024-04-17T17:14:02.291Z{"msg":"Next flush: 1659","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.774019497Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"":"downstairs","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1812024-04-17T17:14:02.291Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.774047382Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"":"downstairs","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1822024-04-17T17:14:02.291Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.774081127Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1832024-04-17T17:14:02.291Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.774106053Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"client":"0","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1842024-04-17T17:14:02.291Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.77412577Z","Region: es:10 ec:5 bs:4096 ts:204800 tb:50 max_io:50 or 204800
1852024-04-17T17:14:02.291Zhostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"client":"1","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1862024-04-17T17:14:02.291Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.774161614Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"client":"2","":"downstairs","session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1872024-04-17T17:14:02.291ZFill test
1882024-04-17T17:14:02.291Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.774194469Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1892024-04-17T17:14:02.291Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.774224674Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1902024-04-17T17:14:02.291Z{"msg":"ead2ccf7-0d55-4a5f-931b-5837bde4d520 is now active with session: 97bcd462-ff38-40bd-9ab5-6ce1e1b72990","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.774255219Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1912024-04-17T17:14:02.291Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.774284795Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206,"session_id":"97bcd462-ff38-40bd-9ab5-6ce1e1b72990"}
1922024-04-17T17:14:02.291Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:00.77431658Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206}
1932024-04-17T17:14:02.326ZRead and Verify all blocks (0..50 range:false)
1942024-04-17T17:14:02.358ZWrote out file "/var/tmp/test_up/verify_file"
1952024-04-17T17:14:02.358ZCLIENT: Tests done. All submitted work has been ACK'd
1962024-04-17T17:14:02.362Z----------------------------------------------------------------
1972024-04-17T17:14:02.362Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
1982024-04-17T17:14:02.362ZStates: Active Active Active
1992024-04-17T17:14:02.362ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
2002024-04-17T17:14:02.362Z 3 Acked 1002 Read 50 Done Done Done false
2012024-04-17T17:14:02.362Z STATES DS:0 DS:1 DS:2 TOTAL
2022024-04-17T17:14:02.362Z New 0 0 0 0
2032024-04-17T17:14:02.362Z Sent 0 0 0 0
2042024-04-17T17:14:02.362Z Done 1 1 1 3
2052024-04-17T17:14:02.362Z Skipped 0 0 0 0
2062024-04-17T17:14:02.362Z Error 0 0 0 0
2072024-04-17T17:14:02.362ZLast Flush: 1001 1001 1001
2082024-04-17T17:14:02.362ZDownstairs last five completed: 1001 1000
2092024-04-17T17:14:02.362ZUpstairs last five completed: 3 2 1
2102024-04-17T17:14:02.362ZCLIENT: Up:0 ds:1 act:3
2112024-04-17T17:14:06.361Z----------------------------------------------------------------
2122024-04-17T17:14:06.362Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2132024-04-17T17:14:06.362ZDownstairs last five completed: 1003 1002 1001 1000
2142024-04-17T17:14:06.362ZUpstairs last five completed: 4 3 2 1
2152024-04-17T17:14:06.362ZCLIENT: Up:0 ds:0 act:3
2162024-04-17T17:14:06.362ZCLIENT: All crucible jobs finished, exiting program
2172024-04-17T17:14:06.365Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:14:04.852700547Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1206}
2182024-04-17T17:14:06.379ZRepair setup passed
2192024-04-17T17:14:06.379ZCopy the 8830 file
2202024-04-17T17:14:06.379Zcp -r /var/tmp/test_up/8830 /var/tmp/test_up/previous
2212024-04-17T17:14:06.379Z/var/tmp/bins/crutest repair -g 32 -q --key GAHk0Qm9zkd65cUouA2Rh9epFmhVPivSi1D/01vBwgc= -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --verify-out /var/tmp/test_up/verify_file --verify-in /var/tmp/test_up/verify_file
2222024-04-17T17:14:06.388Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.87826264Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212}
2232024-04-17T17:14:06.389Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.878498103Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212}
2242024-04-17T17:14:06.408Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.878528848Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212}
2252024-04-17T17:14:06.408Z{"msg":"Crucible 978b1cc2-81ec-499a-8ac3-8ffcb190b2e7 has session id: aad08b05-972d-40c7-aed8-d5854cba6cae","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.878554784Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2262024-04-17T17:14:06.408Z{"msg":"Upstairs opts: Upstairs UUID: 978b1cc2-81ec-499a-8ac3-8ffcb190b2e7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.878595018Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2272024-04-17T17:14:06.408Z{"msg":"Crucible stats registered with UUID: 978b1cc2-81ec-499a-8ac3-8ffcb190b2e7Crucible runtime is spawned
2282024-04-17T17:14:06.408Z","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.878652358Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2292024-04-17T17:14:06.408Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.878693462Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212}
2302024-04-17T17:14:06.408Z{"msg":"978b1cc2-81ec-499a-8ac3-8ffcb190b2e7 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.878799315Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2312024-04-17T17:14:06.408Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.878841529Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"client":"0","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2322024-04-17T17:14:06.408Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.878877083Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"client":"0","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2332024-04-17T17:14:06.409Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.878924286Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"client":"1","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2342024-04-17T17:14:06.409Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.878952541Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"client":"1","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2352024-04-17T17:14:06.409Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.878982856Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"client":"2","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2362024-04-17T17:14:06.409Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.879015711Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"client":"2","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2372024-04-17T17:14:06.409Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.879067213Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"":"io task","client":"2","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2382024-04-17T17:14:06.409Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.879192363Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"":"io task","client":"0","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2392024-04-17T17:14:06.409Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.879222339Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"":"io task","client":"1","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2402024-04-17T17:14:06.409Z{"msg":"ds_connection connected from Ok(127.0.0.1:59797)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.879256153Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"":"io task","client":"0","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2412024-04-17T17:14:06.409Z{"msg":"ds_connection connected from Ok(127.0.0.1:56473)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.87927743Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"":"io task","client":"1","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2422024-04-17T17:14:06.409Z{"msg":"ds_connection connected from Ok(127.0.0.1:46482)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.879307265Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"":"io task","client":"2","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2432024-04-17T17:14:06.409Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.879810486Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"client":"1","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2442024-04-17T17:14:06.409Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.87991199Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"client":"1","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2452024-04-17T17:14:06.409Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.880018123Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"client":"2","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2462024-04-17T17:14:06.409Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.880077743Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"client":"2","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2472024-04-17T17:14:06.409Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.880112898Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"client":"0","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2482024-04-17T17:14:06.410Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.880138444Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"client":"0","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2492024-04-17T17:14:06.410Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.880377636Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"client":"1","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2502024-04-17T17:14:06.410Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.88041913Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"client":"0","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2512024-04-17T17:14:06.410Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.880445675Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"client":"2","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2522024-04-17T17:14:06.410Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.880833184Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"client":"1","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2532024-04-17T17:14:06.410Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.880869938Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"client":"1","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2542024-04-17T17:14:06.410Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.880885986Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2552024-04-17T17:14:06.410Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.880898324Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"client":"2","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2562024-04-17T17:14:06.410Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.880911552Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"client":"2","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2572024-04-17T17:14:06.410Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.880932698Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2582024-04-17T17:14:06.410Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.880945086Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"client":"0","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2592024-04-17T17:14:06.410Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.880957914Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"client":"0","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2602024-04-17T17:14:06.410Z{"msg":"[0]R flush_numbers: [1659, 1659, 1659, 1659, 1659]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.880978531Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"":"downstairs","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2612024-04-17T17:14:06.410Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.880992699Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"":"downstairs","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2622024-04-17T17:14:06.411ZWait for a query_work_queue command to finish before sending IO
2632024-04-17T17:14:06.411Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.881032673Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"":"downstairs","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2642024-04-17T17:14:06.411Z{"msg":"[1]R flush_numbers: [1659, 1659, 1659, 1659, 1659]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.88104748Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"":"downstairs","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2652024-04-17T17:14:06.411Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.881068337Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"":"downstairs","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2662024-04-17T17:14:06.411Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.881081965Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"":"downstairs","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2672024-04-17T17:14:06.411Z{"msg":"[2]R flush_numbers: [1659, 1659, 1659, 1659, 1659]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.881094943Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"":"downstairs","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2682024-04-17T17:14:06.411Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.881120459Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"":"downstairs","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2692024-04-17T17:14:06.411Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.881140425Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"":"downstairs","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2702024-04-17T17:14:06.411Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.881163642Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"":"downstairs","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2712024-04-17T17:14:06.411Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.881191787Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"":"downstairs","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2722024-04-17T17:14:06.411Z{"msg":"Next flush: 1660","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.881205305Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"":"downstairs","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2732024-04-17T17:14:06.411Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.881226942Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"":"downstairs","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2742024-04-17T17:14:06.411Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.881250888Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2752024-04-17T17:14:06.411Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.881266716Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"client":"0","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2762024-04-17T17:14:06.411Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.881289372Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"client":"1","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2772024-04-17T17:14:06.411Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.881313148Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"client":"2","":"downstairs","session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2782024-04-17T17:14:06.411Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.881338314Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2792024-04-17T17:14:06.411Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.881355902Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2802024-04-17T17:14:06.411ZRegion: es:10 ec:5 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2812024-04-17T17:14:06.411Z{"msg":"978b1cc2-81ec-499a-8ac3-8ffcb190b2e7 is now active with session: aad08b05-972d-40c7-aed8-d5854cba6cae","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.881386937Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2822024-04-17T17:14:06.411Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.881401334Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212,"session_id":"aad08b05-972d-40c7-aed8-d5854cba6cae"}
2832024-04-17T17:14:06.411Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:04.881420651Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212}
2842024-04-17T17:14:06.411ZLoading write count information from file "/var/tmp/test_up/verify_file"
2852024-04-17T17:14:06.411ZRun Repair workload
2862024-04-17T17:14:06.412Z01/10 Write block 29 len 24576 data: 2 2 2 2 2 2
2872024-04-17T17:14:06.412Z02/10 Write block 41 len 4096 data: 2
2882024-04-17T17:14:06.412Z03/10 Read block 3 len 12288
2892024-04-17T17:14:06.412Z04/10 Read block 45 len 4096
2902024-04-17T17:14:06.417Z05/10 Read block 19 len 16384
2912024-04-17T17:14:06.417Z06/10 Flush
2922024-04-17T17:14:06.417Z07/10 Write block 40 len 24576 data: 2 3 2 2 2 2
2932024-04-17T17:14:06.417Z08/10 Read block 17 len 4096
2942024-04-17T17:14:06.417Z09/10 Read block 39 len 8192
2952024-04-17T17:14:06.417Z10/10 Write block 13 len 40960 data: 2 2 2 2 2 2 2 2 2 2
2962024-04-17T17:14:06.423Z----------------------------------------------------------------
2972024-04-17T17:14:06.423Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:4
2982024-04-17T17:14:06.423ZStates: Active Active Active
2992024-04-17T17:14:06.423ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
3002024-04-17T17:14:06.423Z 7 Acked 1006 Write 6 Done Done Done false
3012024-04-17T17:14:06.423Z 8 Acked 1007 Read 1 Done Done Done false
3022024-04-17T17:14:06.424Z 9 Acked 1008 Read 2 Done Done Done false
3032024-04-17T17:14:06.424Z 10 Acked 1009 Write 10 Sent Sent Sent false
3042024-04-17T17:14:06.424Z STATES DS:0 DS:1 DS:2 TOTAL
3052024-04-17T17:14:06.424Z New 0 0 0 0
3062024-04-17T17:14:06.424Z Sent 1 1 1 3
3072024-04-17T17:14:06.424Z Done 3 3 3 9
3082024-04-17T17:14:06.424Z Skipped 0 0 0 0
3092024-04-17T17:14:06.424Z Error 0 0 0 0
3102024-04-17T17:14:06.424ZLast Flush: 1005 1005 1005
3112024-04-17T17:14:06.428ZDownstairs last five completed: 1005 1004 1003 1002 1001
3122024-04-17T17:14:06.428ZUpstairs last five completed: 10 9 8 7 6
3132024-04-17T17:14:06.428Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:14:04.915069337Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1212}
3142024-04-17T17:14:06.428ZWrote out file "/var/tmp/test_up/verify_file"
3152024-04-17T17:14:06.428ZRepair part 1 passed
3162024-04-17T17:14:06.428Z
3172024-04-17T17:14:06.428ZKill the current downstairs
3182024-04-17T17:14:06.530Zrm -rf /var/tmp/test_up/8830
3192024-04-17T17:14:06.530ZNow put back the original so we have a mismatch
3202024-04-17T17:14:06.530Zmv /var/tmp/test_up/previous /var/tmp/test_up/8830
3212024-04-17T17:14:06.534ZRestart downstairs with old directory
3222024-04-17T17:14:06.585Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up/8810 -d /var/tmp/test_up/8820 -d /var/tmp/test_up/8830
3232024-04-17T17:14:06.597ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3242024-04-17T17:14:06.601Z 0 000-009 31 31 31 1659 1659 1659 F F F
3252024-04-17T17:14:06.601Z 1 010-019 31 31 31 1659 1659 1659 T T F <---
3262024-04-17T17:14:06.601Z 2 020-029 32 32 31 1660 1660 1659 T T F <---
3272024-04-17T17:14:06.601Z 3 030-039 32 32 31 1660 1660 1659 F F F <---
3282024-04-17T17:14:06.601Z 4 040-049 32 32 31 1660 1660 1659 T T F <---
3292024-04-17T17:14:06.601ZMax gen: 32, Max flush: 1660
3302024-04-17T17:14:06.601ZError: Difference in extent metadata found!
3312024-04-17T17:14:06.601Zdump test found error as expected
3322024-04-17T17:14:06.601Z
3332024-04-17T17:14:06.601Z
3342024-04-17T17:14:06.602Z/var/tmp/bins/crutest deactivate --range -g 33 -q --key GAHk0Qm9zkd65cUouA2Rh9epFmhVPivSi1D/01vBwgc= -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --verify-out /var/tmp/test_up/verify_file --verify-in /var/tmp/test_up/verify_file
3352024-04-17T17:14:06.612Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:05.104172149Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219}
3362024-04-17T17:14:06.612Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:05.104462544Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219}
3372024-04-17T17:14:06.621Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:05.104493099Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219}
3382024-04-17T17:14:06.622Z{"msg":"Crucible 825b7d5d-4394-4054-99f0-de08ba637240 has session id: a06bf45e-2ce8-48d4-8cac-9388ec16a576","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:05.104513236Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3392024-04-17T17:14:06.622Z{"msg":"Upstairs opts: Upstairs UUID: 825b7d5d-4394-4054-99f0-de08ba637240, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:05.104570656Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3402024-04-17T17:14:06.622Z{"msg":"Crucible stats registered with UUID: 825b7d5d-4394-4054-99f0-de08ba637240","v":0,"name":"crucible","level":30Crucible runtime is spawned
3412024-04-17T17:14:06.622Z,"time":"2024-04-17T17:14:05.104604641Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3422024-04-17T17:14:06.622Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:05.104702975Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219}
3432024-04-17T17:14:06.622Z{"msg":"825b7d5d-4394-4054-99f0-de08ba637240 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:05.10473664Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3442024-04-17T17:14:06.622Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:05.104752328Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"0","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3452024-04-17T17:14:06.622Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:05.104773644Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"0","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3462024-04-17T17:14:06.622Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:05.104817208Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"1","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3472024-04-17T17:14:06.622Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:05.104844513Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"1","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3482024-04-17T17:14:06.622Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:05.104884287Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"2","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3492024-04-17T17:14:06.622Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:05.104906713Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"2","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3502024-04-17T17:14:06.622Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:05.104936129Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"":"io task","client":"2","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3512024-04-17T17:14:06.622Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:05.105005588Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"":"io task","client":"0","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3522024-04-17T17:14:06.622Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:05.105032214Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"":"io task","client":"1","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3532024-04-17T17:14:06.622Z{"msg":"ds_connection connected from Ok(127.0.0.1:52261)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:05.105056919Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"":"io task","client":"0","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3542024-04-17T17:14:06.622Z{"msg":"ds_connection connected from Ok(127.0.0.1:52345)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:05.105079656Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"":"io task","client":"1","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3552024-04-17T17:14:06.622Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:14:05.105224313Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"":"io task","client":"2","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3562024-04-17T17:14:06.622Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:14:05.105274935Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"":"io task","client":"2","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3572024-04-17T17:14:06.622Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:14:05.105331906Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3582024-04-17T17:14:06.623Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:05.105379219Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"2","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3592024-04-17T17:14:06.623Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:05.105455696Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"2","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3602024-04-17T17:14:06.623Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:05.105490491Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"":"io task","client":"2","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3612024-04-17T17:14:06.623Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:05.105791603Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"0","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3622024-04-17T17:14:06.623Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:05.105889988Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"0","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3632024-04-17T17:14:06.623Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:05.10594023Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"1","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3642024-04-17T17:14:06.623Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:05.105975214Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"1","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3652024-04-17T17:14:06.623Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:05.106378531Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"0","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3662024-04-17T17:14:06.623Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:05.106534656Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"1","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3672024-04-17T17:14:06.623Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:05.106675364Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"0","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3682024-04-17T17:14:06.623Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:05.106748192Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"0","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3692024-04-17T17:14:06.623Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:05.106775248Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3702024-04-17T17:14:06.623Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:05.106806393Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"1","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3712024-04-17T17:14:06.623Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:05.10683029Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"1","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3722024-04-17T17:14:06.623Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:05.106860825Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3732024-04-17T17:14:16.618Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.107484017Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"":"io task","client":"2","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3742024-04-17T17:14:16.625Z{"msg":"ds_connection connected from Ok(127.0.0.1:51680)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.107598908Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"":"io task","client":"2","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3752024-04-17T17:14:16.625Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.10854127Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"2","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3762024-04-17T17:14:16.625Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.10860306Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"2","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3772024-04-17T17:14:16.625Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.109089513Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"2","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3782024-04-17T17:14:16.625Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.109455765Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"2","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3792024-04-17T17:14:16.625Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.109521885Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"2","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3802024-04-17T17:14:16.625Z{"msg":"[0]R flush_numbers: [1659, 1659, 1660, 1660, 1660]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.109559209Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3812024-04-17T17:14:16.625Z{"msg":"[0]R generation: [31, 31, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.109592134Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3822024-04-17T17:14:16.625Z{"msg":"[0]R dirty: [false, true, true, false, true]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.10961644Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3832024-04-17T17:14:16.625Z{"msg":"[1]R flush_numbers: [1659, 1659, 1660, 1660, 1660]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.109648375Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3842024-04-17T17:14:16.625Z{"msg":"[1]R generation: [31, 31, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.10967949Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3852024-04-17T17:14:16.625Z{"msg":"[1]R dirty: [false, true, true, false, true]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.109710925Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3862024-04-17T17:14:16.625Z{"msg":"[2]R flush_numbers: [1659, 1659, 1659, 1659, 1659]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.10974203Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3872024-04-17T17:14:16.625Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.109773255Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3882024-04-17T17:14:16.625Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.10980422Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3892024-04-17T17:14:16.625Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.109834275Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3902024-04-17T17:14:16.625Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.109864441Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3912024-04-17T17:14:16.625Z{"msg":"Next flush: 1661","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.109887397Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3922024-04-17T17:14:16.626Z{"msg":"Extents 1 dirty","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.109917322Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3932024-04-17T17:14:16.626Z{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.109948757Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3942024-04-17T17:14:16.626Z{"msg":"extent:1 gens: 31 31 31","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.109981452Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3952024-04-17T17:14:16.626Z{"msg":"extent:1 flush: 1659 1659 1659 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.110009088Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3962024-04-17T17:14:16.626Z{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.11005629Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3972024-04-17T17:14:16.626Z{"msg":"extent:1 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.110090395Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3982024-04-17T17:14:16.626Z{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.11012333Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
3992024-04-17T17:14:16.626Z{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.110156395Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4002024-04-17T17:14:16.626Z{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.110189369Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4012024-04-17T17:14:16.626Z{"msg":"Extents 2 dirty","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.110215895Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4022024-04-17T17:14:16.626Z{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.1102473Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4032024-04-17T17:14:16.626Z{"msg":"extent:2 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.110279665Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4042024-04-17T17:14:16.626Z{"msg":"extent:2 flush: 1660 1660 1659 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.11031341Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4052024-04-17T17:14:16.626Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.110346354Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4062024-04-17T17:14:16.626Z{"msg":"extent:2 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.110378929Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4072024-04-17T17:14:16.626Z{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.110411664Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4082024-04-17T17:14:16.626Z{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.110444219Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4092024-04-17T17:14:16.626Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.110487672Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4102024-04-17T17:14:16.626Z{"msg":"Extents 4 dirty","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.110541474Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4112024-04-17T17:14:16.626Z{"msg":"First source client ID for extent 4","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.110593555Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4122024-04-17T17:14:16.626Z{"msg":"extent:4 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.110646417Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4132024-04-17T17:14:16.626Z{"msg":"extent:4 flush: 1660 1660 1659 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.110698879Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4142024-04-17T17:14:16.626Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.110751061Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4152024-04-17T17:14:16.627Z{"msg":"extent:4 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.110804172Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4162024-04-17T17:14:16.627Z{"msg":"find dest for source 0 for extent at index 4","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.110849735Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4172024-04-17T17:14:16.627Z{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.11088181Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4182024-04-17T17:14:16.627Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.110915505Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4192024-04-17T17:14:16.627Z{"msg":"Extent 3 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.11094801Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4202024-04-17T17:14:16.627Z{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.110980445Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4212024-04-17T17:14:16.627Z{"msg":"extent:3 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.111019728Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4222024-04-17T17:14:16.627Z{"msg":"extent:3 flush: 1660 1660 1659 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.111038665Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4232024-04-17T17:14:16.627Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.111055173Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4242024-04-17T17:14:16.627Z{"msg":"extent:3 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.111085978Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4252024-04-17T17:14:16.627Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.111107054Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4262024-04-17T17:14:16.627Z{"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.11113738Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4272024-04-17T17:14:16.627Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.111166125Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4282024-04-17T17:14:16.627Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.1112003Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"0","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4292024-04-17T17:14:16.627Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.111230185Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"1","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4302024-04-17T17:14:16.627Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.11126275Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"2","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4312024-04-17T17:14:16.627Z{"msg":"starting reconciliation 88613151-dad0-4ea3-9674-4a47cefc44e6: found 4 extents that need repair","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.111295265Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4322024-04-17T17:14:16.627Z{"msg":"Full repair list: {4: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, 3: ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, 1: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, 2: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.111330249Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4332024-04-17T17:14:16.627Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: 4, client_id: ClientId(0), flush_number: 1661, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: 4 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: 4, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: 4 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: 3, client_id: ClientId(0), flush_number: 1661, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: 3 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: 3, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: 3 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: 1, client_id: ClientId(0), flush_number: 1661, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: 1 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: 1, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: 1 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: 2, client_id: ClientId(0), flush_number: 1661, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: 2, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: 2 }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.111443301Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4342024-04-17T17:14:16.631Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.112001403Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"1","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4352024-04-17T17:14:16.631Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.112048766Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"2","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4362024-04-17T17:14:16.697Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.184348746Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"2","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4372024-04-17T17:14:16.747Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.237161805Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"1","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4382024-04-17T17:14:16.747Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.237217166Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"2","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4392024-04-17T17:14:16.797Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.287639752Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"1","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4402024-04-17T17:14:16.797Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.287679606Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"2","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4412024-04-17T17:14:16.846Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.335795737Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"":"downstairs","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4422024-04-17T17:14:16.846Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO
4432024-04-17T17:14:16.846Zlevel":30,"time":"2024-04-17T17:14:15.33583556Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"0","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4442024-04-17T17:14:16.847Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.335853237Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"1","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4452024-04-17T17:14:16.847Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.335875474Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"client":"2","":"downstairs","session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4462024-04-17T17:14:16.856Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.335889792Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4472024-04-17T17:14:16.856Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.33590304Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4482024-04-17T17:14:16.856Z{"msg":"825b7d5d-4394-4054-99f0-de08ba637240 is now active with session: a06bf45e-2ce8-48d4-8cac-9388ec16a576","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.335916287Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219,"session_id":"a06bf45e-2ce8-48d4-8cac-9388ec16a576"}
4492024-04-17T17:14:16.857Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:15.335933135Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219}
4502024-04-17T17:14:16.857ZRegion: es:10 ec:5 bs:4096 ts:204800 tb:50 max_io:50 or 204800
4512024-04-17T17:14:16.857ZLoading write count information from file "/var/tmp/test_up/verify_file"
4522024-04-17T17:14:16.857ZRead and Verify all blocks (0..50 range:true)
4532024-04-17T17:14:16.882ZWrote out file "/var/tmp/test_up/verify_file"
4542024-04-17T17:14:16.882ZVerify test completed
4552024-04-17T17:14:16.886ZWrote out file "/var/tmp/test_up/verify_file"
4562024-04-17T17:14:16.886ZCLIENT: Tests done. All submitted work has been ACK'd
4572024-04-17T17:14:16.886Z----------------------------------------------------------------
4582024-04-17T17:14:16.886Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
4592024-04-17T17:14:16.886ZStates: Active Active Active
4602024-04-17T17:14:16.886ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
4612024-04-17T17:14:16.886Z 1 Acked 1000 Read 50 Done Done Done false
4622024-04-17T17:14:16.886Z STATES DS:0 DS:1 DS:2 TOTAL
4632024-04-17T17:14:16.886Z New 0 0 0 0
4642024-04-17T17:14:16.886Z Sent 0 0 0 0
4652024-04-17T17:14:16.886Z Done 1 1 1 3
4662024-04-17T17:14:16.886Z Skipped 0 0 0 0
4672024-04-17T17:14:16.886Z Error 0 0 0 0
4682024-04-17T17:14:16.886ZLast Flush: 0 0 0
4692024-04-17T17:14:16.887ZDownstairs last five completed:
4702024-04-17T17:14:16.887ZUpstairs last five completed: 1
4712024-04-17T17:14:16.887ZCLIENT: Up:0 ds:1 act:3
4722024-04-17T17:14:20.885Z----------------------------------------------------------------
4732024-04-17T17:14:20.885Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
4742024-04-17T17:14:20.885ZDownstairs last five completed: 1001 1000
4752024-04-17T17:14:20.885ZUpstairs last five completed: 2 1
4762024-04-17T17:14:20.885ZCLIENT: Up:0 ds:0 act:3
4772024-04-17T17:14:20.885ZCLIENT: All crucible jobs finished, exiting program
4782024-04-17T17:14:20.888Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:14:19.374417804Z","hostname":"w-01HVPGE0ZWY84ZKN7SCZ1H0XWF","pid":1219}
4792024-04-17T17:14:20.892ZRepair part 2 passed
4802024-04-17T17:14:20.892Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up/8810 -d /var/tmp/test_up/8820 -d /var/tmp/test_up/8830
4812024-04-17T17:14:20.902ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4822024-04-17T17:14:20.902Z 0 000-009 31 31 31 1659 1659 1659 F F F
4832024-04-17T17:14:20.903Z 1 010-019 33 33 33 1661 1661 1661 F F F
4842024-04-17T17:14:20.903Z 2 020-029 33 33 33 1661 1661 1661 F F F
4852024-04-17T17:14:20.903Z 3 030-039 32 32 32 1660 1660 1660 F F F
4862024-04-17T17:14:20.903Z 4 040-049 33 33 33 1661 1661 1661 F F F
4872024-04-17T17:14:20.903ZMax gen: 33, Max flush: 1661
4882024-04-17T17:14:20.906Zdump test passed
4892024-04-17T17:14:20.907Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up/8810 -d /var/tmp/test_up/8820 -d /var/tmp/test_up/8830 -e 1
4902024-04-17T17:14:20.915Z Extent 1
4912024-04-17T17:14:20.915ZGEN 33 33 33
4922024-04-17T17:14:20.915ZFLUSH_ID 1661 1661 1661
4932024-04-17T17:14:20.915ZDIRTY
4942024-04-17T17:14:20.915Z
4952024-04-17T17:14:20.915ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
4962024-04-17T17:14:20.920Z010 A A A A A A
4972024-04-17T17:14:20.921Z011 A A A A A A
4982024-04-17T17:14:20.921Z012 A A A A A A
4992024-04-17T17:14:20.921Z013 A A A A A A
5002024-04-17T17:14:20.921Z014 A A A A A A
5012024-04-17T17:14:20.921Z015 A A A A A A
5022024-04-17T17:14:20.922Z016 A A A A A A
5032024-04-17T17:14:20.922Z017 A A A A A A
5042024-04-17T17:14:20.922Z018 A A A A A A
5052024-04-17T17:14:20.922Z019 A A A A A A
5062024-04-17T17:14:20.927Zdump extent test passed
5072024-04-17T17:14:20.927Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up/8810 -d /var/tmp/test_up/8820 -d /var/tmp/test_up/8830 -b 20
5082024-04-17T17:14:20.932ZExtent 2 Block in extent 0 Actual block 20
5092024-04-17T17:14:20.932Z
5102024-04-17T17:14:20.936Z DATA SHA256 VER
5112024-04-17T17:14:20.936Z------ ---------------------------------------------------------------- ---
5122024-04-17T17:14:20.936Z 0 90b7efe16d0d8f0c5df70f016113e5ac6656c31b1b4ac73daa9e7693a00c5ca4 A
5132024-04-17T17:14:20.936Z 1 90b7efe16d0d8f0c5df70f016113e5ac6656c31b1b4ac73daa9e7693a00c5ca4 A
5142024-04-17T17:14:20.936Z 2 90b7efe16d0d8f0c5df70f016113e5ac6656c31b1b4ac73daa9e7693a00c5ca4 A
5152024-04-17T17:14:20.936Z
5162024-04-17T17:14:20.936ZNONCES 0 1 2 DIFF
5172024-04-17T17:14:20.936Z------ ------------------------ ------------------------ ------------------------ -----
5182024-04-17T17:14:20.936Z 0 16ea07077fdb566e25bdf1fe 16ea07077fdb566e25bdf1fe 16ea07077fdb566e25bdf1fe
5192024-04-17T17:14:20.936Z
5202024-04-17T17:14:20.936Z TAGS 0 1 2 DIFF
5212024-04-17T17:14:20.937Z------ -------------------------------- -------------------------------- -------------------------------- -----
5222024-04-17T17:14:20.937Z 0 96a5c3a8d1476c5f248c9072c773f957 96a5c3a8d1476c5f248c9072c773f957 96a5c3a8d1476c5f248c9072c773f957
5232024-04-17T17:14:20.937Z
5242024-04-17T17:14:20.937ZHASHES 0 1 2 DIFF
5252024-04-17T17:14:20.937Z------ ---------------- ---------------- ---------------- -----
5262024-04-17T17:14:20.937Z 0 1aefed5f2cb6c00c 1aefed5f2cb6c00c 1aefed5f2cb6c00c
5272024-04-17T17:14:20.937Z
5282024-04-17T17:14:20.943Zdump block test passed
5292024-04-17T17:14:20.943ZUpstairs tests have completed, stopping all downstairs
5302024-04-17T17:14:20.991Z
5312024-04-17T17:14:20.991ZAll Tests have passed
5322024-04-17T17:14:21.003Z3:34 Test duration
5332024-04-17T17:14:21.003Z
5342024-04-17T17:14:21.004Zreal 3:33.541605433
5352024-04-17T17:14:21.004Zuser 2:21.984986286
5362024-04-17T17:14:21.004Zsys 6.900955525
5372024-04-17T17:14:21.004Ztrap 0.162055724
5382024-04-17T17:14:21.004Ztflt 0.010828059
5392024-04-17T17:14:21.004Zdflt 0.109136218
5402024-04-17T17:14:21.004Zkflt 0.001347391
5412024-04-17T17:14:21.004Zlock 51:20.893045772
5422024-04-17T17:14:21.004Zslp 6:56.996724866
5432024-04-17T17:14:21.004Zlat 14.748824193
5442024-04-17T17:14:21.004Zstop 0.031948686
5452024-04-17T17:14:21.004Zprocess exited: duration 221059 ms, exit code 0
 
5462024-04-17T17:14:21.018Zfound 5 output files
5472024-04-17T17:14:21.018Zuploading: /tmp/test_up/dsc-out.txt (4484 bytes)
5482024-04-17T17:14:25.895Zuploaded: /tmp/test_up/dsc-out.txt
5492024-04-17T17:14:25.903Zuploading: /tmp/test_up/test_up_out.txt (4043511 bytes)
5502024-04-17T17:14:28.740Zuploaded: /tmp/test_up/test_up_out.txt
5512024-04-17T17:14:28.740Zuploading: /tmp/test_up/dsc/downstairs-8810.txt (55907 bytes)
5522024-04-17T17:14:29.756Zuploaded: /tmp/test_up/dsc/downstairs-8810.txt
5532024-04-17T17:14:29.757Zuploading: /tmp/test_up/dsc/downstairs-8820.txt (50565 bytes)
5542024-04-17T17:14:30.775Zuploaded: /tmp/test_up/dsc/downstairs-8820.txt
5552024-04-17T17:14:30.775Zuploading: /tmp/test_up/dsc/downstairs-8830.txt (9914 bytes)
5562024-04-17T17:14:31.790Zuploaded: /tmp/test_up/dsc/downstairs-8830.txt