01J8RRNCZBD8FD9MS13EDNWBNT: test-up-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up/*.txt",
        "%/tmp/test_up/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01J8RRNVNE90ZTVKW3DMPPKJ02

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-09-27T03:49:00.460Zjob dependencies complete; ready to run (waiting for 14 m 58 s)
22024-09-27T03:49:57.862Zjob assigned to worker 01J8RSHJ0DQD1FAYE3AWJ6AGZ9 (queued for 57 s)
32024-09-27T03:50:05.797Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-09-27T03:50:06.879Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-09-27T03:50:06.879Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-09-27T03:50:07.936Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-09-27T03:50:07.936Zdownloading input: /input/build/work/bins/crutest.gz
82024-09-27T03:50:09.029Zdownloaded input: /input/build/work/bins/crutest.gz
92024-09-27T03:50:09.029Zdownloading input: /input/build/work/bins/dsc.gz
102024-09-27T03:50:09.685Zdownloaded input: /input/build/work/bins/dsc.gz
112024-09-27T03:50:09.688Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-09-27T03:50:09.703Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-09-27T03:50:09.704Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-09-27T03:50:09.724Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-09-27T03:50:09.724Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-09-27T03:50:09.742Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-09-27T03:50:09.742Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-09-27T03:50:09.760Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-09-27T03:50:09.760Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-09-27T03:50:09.782Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-09-27T03:50:09.782Zdownloading input: /input/build/work/scripts/test_up.sh
222024-09-27T03:50:09.805Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-09-27T03:50:09.805Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-09-27T03:50:09.822Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-09-27T03:50:09.825Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-09-27T03:50:10.592Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-09-27T03:50:10.592Zstarting task 0: "setup"
282024-09-27T03:50:10.602Z++ uname -s
292024-09-27T03:50:10.604Z+ kern=SunOS
302024-09-27T03:50:10.604Z+ case "$kern" in
312024-09-27T03:50:10.605Z+ groupadd -g 12345 build
322024-09-27T03:50:10.607Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-09-27T03:50:12.617Z+ zfs create -o mountpoint=/work rpool/work
342024-09-27T03:50:12.858Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-09-27T03:50:12.860Z+ home_fs=zfs
362024-09-27T03:50:12.861Z+ [[ zfs == autofs ]]
372024-09-27T03:50:12.861Z+ mkdir -p /home/build
382024-09-27T03:50:12.863Z+ chown build:build /home/build /work
392024-09-27T03:50:14.866Z+ chmod 0700 /home/build /work
402024-09-27T03:50:14.869Zprocess exited: duration 4274 ms, exit code 0
 
412024-09-27T03:50:14.875Zstarting task 1: "authentication"
422024-09-27T03:50:14.892Zprocess exited: duration 16 ms, exit code 0
 
432024-09-27T03:50:14.897Zstarting task 2: "build"
442024-09-27T03:50:14.902Z+ banner cores
452024-09-27T03:50:14.925Z
462024-09-27T03:50:14.925Z #### #### ##### ###### ####
472024-09-27T03:50:14.925Z # # # # # # # #
482024-09-27T03:50:14.925Z # # # # # ##### ####
492024-09-27T03:50:14.925Z # # # ##### # #
502024-09-27T03:50:14.925Z # # # # # # # # #
512024-09-27T03:50:14.925Z #### #### # # ###### ####
522024-09-27T03:50:14.925Z
532024-09-27T03:50:14.928Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-09-27T03:50:14.937Z+ echo 'input bins dir contains:'
552024-09-27T03:50:14.940Zinput bins dir contains:
562024-09-27T03:50:14.940Z+ ls -ltr /input/build/work/bins
572024-09-27T03:50:14.940Ztotal 751246
582024-09-27T03:50:14.940Z-rw-r--r-- 1 root root 108131710 Sep 27 03:50 crucible-downstairs.gz
592024-09-27T03:50:14.940Z-rw-r--r-- 1 root root 102511635 Sep 27 03:50 crucible-hammer.gz
602024-09-27T03:50:14.940Z-rw-r--r-- 1 root root 108522272 Sep 27 03:50 crutest.gz
612024-09-27T03:50:14.940Z-rw-r--r-- 1 root root 65218862 Sep 27 03:50 dsc.gz
622024-09-27T03:50:14.940Z+ banner unpack
632024-09-27T03:50:14.946Z
642024-09-27T03:50:14.946Z # # # # ##### ## #### # #
652024-09-27T03:50:14.946Z # # ## # # # # # # # # #
662024-09-27T03:50:14.946Z # # # # # # # # # # ####
672024-09-27T03:50:14.946Z # # # # # ##### ###### # # #
682024-09-27T03:50:14.946Z # # # ## # # # # # # #
692024-09-27T03:50:14.946Z #### # # # # # #### # #
702024-09-27T03:50:14.946Z
712024-09-27T03:50:14.946Z+ mkdir -p /var/tmp/bins
722024-09-27T03:50:14.948Z+ for t in "$input/bins/"*.gz
732024-09-27T03:50:14.949Z++ basename /input/build/work/bins/crucible-downstairs.gz
742024-09-27T03:50:14.949Z+ b=crucible-downstairs.gz
752024-09-27T03:50:14.949Z+ b=crucible-downstairs
762024-09-27T03:50:14.949Z+ gunzip
772024-09-27T03:50:17.221Z+ chmod +x /var/tmp/bins/crucible-downstairs
782024-09-27T03:50:17.225Z+ for t in "$input/bins/"*.gz
792024-09-27T03:50:17.225Z++ basename /input/build/work/bins/crucible-hammer.gz
802024-09-27T03:50:17.228Z+ b=crucible-hammer.gz
812024-09-27T03:50:17.228Z+ b=crucible-hammer
822024-09-27T03:50:17.228Z+ gunzip
832024-09-27T03:50:19.368Z+ chmod +x /var/tmp/bins/crucible-hammer
842024-09-27T03:50:19.371Z+ for t in "$input/bins/"*.gz
852024-09-27T03:50:19.373Z++ basename /input/build/work/bins/crutest.gz
862024-09-27T03:50:19.373Z+ b=crutest.gz
872024-09-27T03:50:19.373Z+ b=crutest
882024-09-27T03:50:19.373Z+ gunzip
892024-09-27T03:50:21.702Z+ chmod +x /var/tmp/bins/crutest
902024-09-27T03:50:21.704Z+ for t in "$input/bins/"*.gz
912024-09-27T03:50:21.707Z++ basename /input/build/work/bins/dsc.gz
922024-09-27T03:50:21.707Z+ b=dsc.gz
932024-09-27T03:50:21.707Z+ b=dsc
942024-09-27T03:50:21.707Z+ gunzip
952024-09-27T03:50:23.096Z+ chmod +x /var/tmp/bins/dsc
962024-09-27T03:50:23.116Z+ export BINDIR=/var/tmp/bins
972024-09-27T03:50:23.117Z+ BINDIR=/var/tmp/bins
982024-09-27T03:50:23.117Z+ jobpid=1138
992024-09-27T03:50:23.117Z+ echo 'Setup debug logging'
1002024-09-27T03:50:23.117ZSetup debug logging
1012024-09-27T03:50:23.117Z+ mkdir /tmp/debug
1022024-09-27T03:50:23.117Z+ sleep 7200
1032024-09-27T03:50:23.117Z+ psrinfo -v
1042024-09-27T03:50:23.117Z+ df -h
1052024-09-27T03:50:23.117Z+ prstat -d d -mLc 1
1062024-09-27T03:50:23.117Z+ iostat -T d -xn 1
1072024-09-27T03:50:23.132Z+ mpstat -T d 1
1082024-09-27T03:50:23.132Z+ vmstat -T d -p 1
1092024-09-27T03:50:23.132Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1102024-09-27T03:50:23.132Z+ banner test_up_encrypted
1112024-09-27T03:50:23.133Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1122024-09-27T03:50:23.133Z
1132024-09-27T03:50:23.133Z ##### ###### #### ##### # # ##### ###### # #
1142024-09-27T03:50:23.134Z # # # # # # # # # ## #
1152024-09-27T03:50:23.134Z # ##### #### # # # # # ##### # # #
1162024-09-27T03:50:23.134Z # # # # # # ##### # # # #
1172024-09-27T03:50:23.134Z # # # # # # # # # # ##
1182024-09-27T03:50:23.134Z # ###### #### # ####### #### # ####### ###### # #
1192024-09-27T03:50:23.134Z
1202024-09-27T03:50:23.134Z+ ptime -m bash /input/build/work/scripts/test_up.sh -N encrypted
1212024-09-27T03:50:23.235Z/input/build/work
1222024-09-27T03:50:23.263ZTurn off color for downstairs dump
1232024-09-27T03:50:23.312ZUpstairs using key: NOZATywXytdQgM+J/eGxj9ahhft21rSFsjcAdbvyhx4=
1242024-09-27T03:50:23.330Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1252024-09-27T03:50:23.330ZCreating three downstairs regions
1262024-09-27T03:50:23.503ZStarting three downstairs
1272024-09-27T03:50:28.514ZDisable automatic restart on all downstairs
1282024-09-27T03:50:28.574Z
1292024-09-27T03:50:28.574ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1302024-09-27T03:50:28.577ZRunning test: span
1312024-09-27T03:50:32.726ZCompleted test: span
1322024-09-27T03:50:37.737ZRunning test: big
1332024-09-27T03:50:42.146ZCompleted test: big
1342024-09-27T03:50:47.156ZRunning test: dep
1352024-09-27T03:51:41.973ZCompleted test: dep
1362024-09-27T03:51:46.985ZRunning test: balloon
1372024-09-27T03:52:33.299ZCompleted test: balloon
1382024-09-27T03:52:38.310ZRunning test: deactivate
1392024-09-27T03:53:33.132ZCompleted test: deactivate
1402024-09-27T03:53:38.142ZRunning hammer
1412024-09-27T03:53:51.711ZRun repair tests
1422024-09-27T03:53:51.722Z/var/tmp/bins/crutest fill -g 31 -q --key NOZATywXytdQgM+J/eGxj9ahhft21rSFsjcAdbvyhx4= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1432024-09-27T03:53:51.762Z{"msg":"use region info: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.353709031Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"task":"crutest"}
1442024-09-27T03:53:51.791Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.354252304Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"task":"crutest"}
1452024-09-27T03:53:51.791Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.357956613Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"task":"crutest"}
1462024-09-27T03:53:51.791Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.358138757Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218}
1472024-09-27T03:53:51.791Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.358186614Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218}
1482024-09-27T03:53:51.791Z{"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.358213075Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218}
1492024-09-27T03:53:51.791Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.358232152Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218}
1502024-09-27T03:53:51.791Z{"msg":"Crucible 2726baa0-e5b8-454c-8a09-a5aeca21a0d5 has session id: 9e64fdaf-632e-4efd-a952-3b2f6f7efc24","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.358734484Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1512024-09-27T03:53:51.791Z{"msg":"Upstairs opts: Upstairs UUID: 2726baa0-e5b8-454c-8a09-a5aeca21a0d5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.358782071Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1522024-09-27T03:53:51.791Z{"msg":"Crucible stats registered with UUID: 2726baa0-e5b8-454c-8a09-a5aeca21a0d5","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.35882588Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1532024-09-27T03:53:51.803Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.392343851Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218}
1542024-09-27T03:53:51.814Z{"msg":"2726baa0-e5b8-454c-8a09-a5aeca21a0d5 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.392484733Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1552024-09-27T03:53:51.814Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.392533659Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"client":"0","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1562024-09-27T03:53:51.814Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.392562699Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"client":"0","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1572024-09-27T03:53:51.814Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.392579377Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"client":"1","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1582024-09-27T03:53:51.814Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.392611015Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"client":"1","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1592024-09-27T03:53:51.814Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.3926339Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"client":"2","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1602024-09-27T03:53:51.814Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.392657073Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"client":"2","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1612024-09-27T03:53:51.814Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.392689681Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"":"io task","client":"2","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1622024-09-27T03:53:51.814Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.392713764Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"":"io task","client":"0","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1632024-09-27T03:53:51.814Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.392754546Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"":"io task","client":"1","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1642024-09-27T03:53:51.814Z{"msg":"ds_connection connected from Ok(127.0.0.1:40173)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.392787653Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"":"io task","client":"2","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1652024-09-27T03:53:51.814Z{"msg":"ds_connection connected from Ok(127.0.0.1:64449)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.39282019Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"":"io task","client":"0","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1662024-09-27T03:53:51.814Z{"msg":"ds_connection connected from Ok(127.0.0.1:43549)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.392856415Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"":"io task","client":"1","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1672024-09-27T03:53:51.814Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.393624212Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"client":"0","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1682024-09-27T03:53:51.814Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.393665104Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"client":"0","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1692024-09-27T03:53:51.814Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.393691985Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"client":"2","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1702024-09-27T03:53:51.814Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.393712911Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"client":"2","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1712024-09-27T03:53:51.815Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.393732078Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"client":"1","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1722024-09-27T03:53:51.815Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.393751504Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"client":"1","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1732024-09-27T03:53:51.815Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.394258622Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"client":"2","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1742024-09-27T03:53:51.815Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.394345542Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"client":"1","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1752024-09-27T03:53:51.815Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.394416512Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"client":"0","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1762024-09-27T03:53:51.815Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.394628865Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"client":"2","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1772024-09-27T03:53:51.815Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.394666549Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"client":"2","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1782024-09-27T03:53:51.815Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.394692421Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1792024-09-27T03:53:51.815Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.39476564Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"client":"1","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1802024-09-27T03:53:51.815Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.394798877Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"client":"1","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1812024-09-27T03:53:51.815Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.39482398Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1822024-09-27T03:53:51.815Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.394853299Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"client":"0","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1832024-09-27T03:53:51.815Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.394887696Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"client":"0","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1842024-09-27T03:53:51.815Z{"msg":"[0]R flush_numbers: [1698, 1698, 1698, 1698, 1698]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.394920673Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"":"downstairs","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1852024-09-27T03:53:51.815Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.394946655Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"":"downstairs","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1862024-09-27T03:53:51.815Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.394979562Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"":"downstairs","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1872024-09-27T03:53:51.815Z{"msg":"[1]R flush_numbers: [1698, 1698, 1698, 1698, 1698]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.395013078Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
1882024-09-27T03:53:51.815Z9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1892024-09-27T03:53:51.815Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.395109911Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"":"downstairs","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1902024-09-27T03:53:51.815Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.395145417Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"":"downstairs","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1912024-09-27T03:53:51.815Z{"msg":"[2]R flush_numbers: [1698, 1698, 1698, 1698, 1698]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.395167871Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"":"downstairs","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1922024-09-27T03:53:51.815Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.395191555Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"":"downstairs","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1932024-09-27T03:53:51.815Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.395222463Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"":"downstairs","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1942024-09-27T03:53:51.816Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.395246436Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"":"downstairs","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1952024-09-27T03:53:51.816Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.39526998Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"":"downstairs","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1962024-09-27T03:53:51.816Z{"msg":"Next flush: 1699","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.395283751Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"":"downstairs","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1972024-09-27T03:53:51.816Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.395304546Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"":"downstairs","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1982024-09-27T03:53:51.816Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.395318147Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
1992024-09-27T03:53:51.816Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.395337373Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"client":"0","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
2002024-09-27T03:53:51.816Z{"msg":"region info:
2012024-09-27T03:53:51.816ZTransition from WaitQuorum to Active" block size: 4096 bytes
2022024-09-27T03:53:51.816Z,"v":0,"name":" sub_volume 0 blocks / extent: 10
2032024-09-27T03:53:51.816Zcrucible","level":30,"time":"2024-09-27T03:53:51.39535735Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"client":"1","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
2042024-09-27T03:53:51.816Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.39537148Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"client":"2","":"downstairs","session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
2052024-09-27T03:53:51.816Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.395392545Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
2062024-09-27T03:53:51.816Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30 sub_volume 0 extent size: 40 KiB
2072024-09-27T03:53:51.816Z, sub_volume 0 extent count: 5
2082024-09-27T03:53:51.816Z" total blocks: 50
2092024-09-27T03:53:51.816Ztime":"2024-09-27T03:53:51.395405197Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218 total size: 200 KiB
2102024-09-27T03:53:51.816Z, encryption: yes
2112024-09-27T03:53:51.816Z"session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
2122024-09-27T03:53:51.816Z{"msg":"2726baa0-e5b8-454c-8a09-a5aeca21a0d5 is now active with session: 9e64fdaf-632e-4efd-a952-3b2f6f7efc24","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.395425442Z","Fill test
2132024-09-27T03:53:51.816Zhostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
2142024-09-27T03:53:51.816Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.395449116Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218,"session_id":"9e64fdaf-632e-4efd-a952-3b2f6f7efc24"}
2152024-09-27T03:53:51.816Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:51.395461267Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218}
2162024-09-27T03:53:51.851ZRead and Verify all blocks (0..50 range:false)
2172024-09-27T03:53:51.892ZWrote out file "/var/tmp/test_up-build/verify_file"
2182024-09-27T03:53:51.892ZCLIENT: Tests done. All submitted work has been ACK'd
2192024-09-27T03:53:51.899Z----------------------------------------------------------------
2202024-09-27T03:53:51.899Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2212024-09-27T03:53:51.899ZStates: Active Active Active
2222024-09-27T03:53:51.899ZGW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2232024-09-27T03:53:51.899Z 3 Acked 1002 Read 50 Done Done Done false
2242024-09-27T03:53:51.899Z STATES DS:0 DS:1 DS:2 TOTAL
2252024-09-27T03:53:51.900Z Sent 0 0 0 0
2262024-09-27T03:53:51.900Z Done 1 1 1 3
2272024-09-27T03:53:51.900Z Skipped 0 0 0 0
2282024-09-27T03:53:51.900Z Error 0 0 0 0
2292024-09-27T03:53:51.900ZLast Flush: 1001 1001 1001
2302024-09-27T03:53:51.900ZDownstairs last five completed: 1001 1000
2312024-09-27T03:53:51.900ZUpstairs last five completed: 3 2 1
2322024-09-27T03:53:51.900ZCLIENT: Up:0 ds:1 act:3
2332024-09-27T03:53:56.740Z----------------------------------------------------------------
2342024-09-27T03:53:56.740Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2352024-09-27T03:53:56.741ZDownstairs last five completed: 1003 1002 1001 1000
2362024-09-27T03:53:56.741ZUpstairs last five completed: 4 3 2 1
2372024-09-27T03:53:56.741ZCLIENT: Up:0 ds:0 act:3
2382024-09-27T03:53:56.741ZCLIENT: All crucible jobs finished, exiting program
2392024-09-27T03:53:56.759Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:53:55.486301863Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1218}
2402024-09-27T03:53:56.759ZRepair setup passed
2412024-09-27T03:53:56.759ZCopy the region for /var/tmp/test_up-build/8830
2422024-09-27T03:53:56.759Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2432024-09-27T03:53:56.759Z/var/tmp/bins/crutest repair -g 32 -q --key NOZATywXytdQgM+J/eGxj9ahhft21rSFsjcAdbvyhx4= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2442024-09-27T03:53:56.774Z{"msg":"use region info: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.563355782Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"task":"crutest"}
2452024-09-27T03:53:56.774Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.56397889Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"task":"crutest"}
2462024-09-27T03:53:56.775Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.565684347Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"task":"crutest"}
2472024-09-27T03:53:56.775Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.565853299Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224}
2482024-09-27T03:53:56.775Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.565905443Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224}
2492024-09-27T03:53:56.775Z{"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.565932554Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224}
2502024-09-27T03:53:56.775Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.565958097Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224}
2512024-09-27T03:53:56.775Z{"msg":"Crucible 6988d2d9-31d9-4aa9-87ff-22acf1b09edb has session id: dd2996c3-7c96-4975-8313-d91e23728a7c","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.566384281Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2522024-09-27T03:53:56.775Z{"msg":"Upstairs opts: Upstairs UUID: 6988d2d9-31d9-4aa9-87ff-22acf1b09edb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.566439492Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2532024-09-27T03:53:56.775Z{"msg":"Crucible stats registered with UUID: 6988d2d9-31d9-4aa9-87ff-22acf1b09edb","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.566465514Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2542024-09-27T03:53:56.776Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.599693986Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224}
2552024-09-27T03:53:56.777Z{"msg":"6988d2d9-31d9-4aa9-87ff-22acf1b09edb active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.599754084Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2562024-09-27T03:53:56.777Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.599853325Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"client":"0","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2572024-09-27T03:53:56.777Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.599909197Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"client":"0","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2582024-09-27T03:53:56.777Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.599969415Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"client":"1","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2592024-09-27T03:53:56.777Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.600030822Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"client":"1","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2602024-09-27T03:53:56.777Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.600053207Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"client":"2","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2612024-09-27T03:53:56.777Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.600090661Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"client":"2","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2622024-09-27T03:53:56.777Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.600128095Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2632024-09-27T03:53:56.777Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.600168127Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2642024-09-27T03:53:56.777Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.600345934Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2652024-09-27T03:53:56.777Z{"msg":"ds_connection connected from Ok(127.0.0.1:36360)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.600376133Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2662024-09-27T03:53:56.777Z{"msg":"ds_connection connected from Ok(127.0.0.1:44227)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.600411658Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2672024-09-27T03:53:56.777Z{"msg":"ds_connection connected from Ok(127.0.0.1:45980)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.600452729Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2682024-09-27T03:53:56.777Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.600964734Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"client":"2","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2692024-09-27T03:53:56.777Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.601033996Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"client":"2","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2702024-09-27T03:53:56.777Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.60107159Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"client":"1","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2712024-09-27T03:53:56.777Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.601113251Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"client":"1","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2722024-09-27T03:53:56.777Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.601150415Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"client":"0","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2732024-09-27T03:53:56.777Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.601169952Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"client":"0","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2742024-09-27T03:53:56.777Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.601581876Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"client":"2","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2752024-09-27T03:53:56.777Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.601724307Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"client":"0","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2762024-09-27T03:53:56.777Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.601765938Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"client":"1","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2772024-09-27T03:53:56.777Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.60196549Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"client":"2","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2782024-09-27T03:53:56.777Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.601991412Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"client":"2","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2792024-09-27T03:53:56.778Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.60203786Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2802024-09-27T03:53:56.778Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.602134053Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"client":"1","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2812024-09-27T03:53:56.778Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.602184188Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"client":"1","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2822024-09-27T03:53:56.778Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.602213198Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2832024-09-27T03:53:56.778Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.602237981Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"client":"0","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2842024-09-27T03:53:56.778Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.602264582Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"client":"0","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2852024-09-27T03:53:56.778Z{"msg":"[0]R flush_numbers: [1699, 1699, 1699, 1699, 1699]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.60229677Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"":"downstairs","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2862024-09-27T03:53:56.779Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.602315217Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"":"downstairs","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2872024-09-27T03:53:56.779Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.60234025Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"":"downstairs","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2882024-09-27T03:53:56.779Z{"msg":"[1]R flush_numbers: [1699, 1699, 1699, 1699, 1699]","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO
2892024-09-27T03:53:56.779Zlevel":30,"time":"2024-09-27T03:53:55.602373347Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"":"downstairs","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2902024-09-27T03:53:56.779Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.602398289Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"":"downstairs","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2912024-09-27T03:53:56.779Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.602422622Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"":"downstairs","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2922024-09-27T03:53:56.779Z{"msg":"[2]R flush_numbers: [1699, 1699, 1699, 1699, 1699]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.602447625Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"":"downstairs","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2932024-09-27T03:53:56.779Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.602472038Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"":"downstairs","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2942024-09-27T03:53:56.779Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.602496421Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"":"downstairs","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2952024-09-27T03:53:56.779Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.602520514Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"":"downstairs","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2962024-09-27T03:53:56.779Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.602544997Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"":"downstairs","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2972024-09-27T03:53:56.779Z{"msg":"Next flush: 1700","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.602568841Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"":"downstairs","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2982024-09-27T03:53:56.779Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.602593244Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"":"downstairs","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
2992024-09-27T03:53:56.779Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.602617537Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
3002024-09-27T03:53:56.779Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.602640011Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"client":"0","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
3012024-09-27T03:53:56.780Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.602664065Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"client":"1","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
3022024-09-27T03:53:56.780Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30region info:
3032024-09-27T03:53:56.780Z block size: 4096 bytes
3042024-09-27T03:53:56.780Z sub_volume 0 blocks / extent: 10
3052024-09-27T03:53:56.780Z,"time":"2024-09-27T03:53:55.602688188Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"client":"2","":"downstairs","session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
3062024-09-27T03:53:56.780Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.60271436Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
3072024-09-27T03:53:56.780Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30, sub_volume 0 extent size: 40 KiB
3082024-09-27T03:53:56.780Z" sub_volume 0 extent count: 5
3092024-09-27T03:53:56.780Ztime total blocks: 50
3102024-09-27T03:53:56.780Z":"2024-09-27T03:53:55.602737534Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid" total size: 200 KiB
3112024-09-27T03:53:56.780Z: encryption: yes
3122024-09-27T03:53:56.780Z1224,"session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
3132024-09-27T03:53:56.780Z{"msg":"6988d2d9-31d9-4aa9-87ff-22acf1b09edb is now active with session: dd2996c3-7c96-4975-8313-d91e23728a7c","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.60277126Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
3142024-09-27T03:53:56.780Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.602794704Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224,"session_id":"dd2996c3-7c96-4975-8313-d91e23728a7c"}
3152024-09-27T03:53:56.780Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.602818947Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224}
3162024-09-27T03:53:56.780ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
3172024-09-27T03:53:56.780ZRun Repair workload
3182024-09-27T03:53:56.780Z01/10 Flush
3192024-09-27T03:53:56.780Z02/10 Write block 2 len 4096 data: 2
3202024-09-27T03:53:56.780Z03/10 Read block 17 len 20480
3212024-09-27T03:53:56.780Z04/10 Write block 9 len 20480 data: 2 2 2 2 2
3222024-09-27T03:53:56.780Z05/10 Read block 31 len 24576
3232024-09-27T03:53:56.780Z06/10 Write block 27 len 36864 data: 2 2 2 2 2 2 2 2 2
3242024-09-27T03:53:56.780Z07/10 Read block 21 len 32768
3252024-09-27T03:53:56.780Z08/10 Write block 3 len 32768 data: 2 2 2 2 2 2 3 3
3262024-09-27T03:53:56.780Z09/10 Read block 37 len 20480
3272024-09-27T03:53:56.780Z10/10 Read block 7 len 24576
3282024-09-27T03:53:56.780Z----------------------------------------------------------------
3292024-09-27T03:53:56.780Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:9
3302024-09-27T03:53:56.780ZStates: Active Active Active
3312024-09-27T03:53:56.780ZGW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
3322024-09-27T03:53:56.780Z 2 Acked 1001 Write 1 Done Done Done false
3332024-09-27T03:53:56.780Z 3 Acked 1002 Read 5 Done Done Done false
3342024-09-27T03:53:56.780Z 4 Acked 1003 Write 5 Done Done Done false
3352024-09-27T03:53:56.780Z 5 Acked 1004 Read 6 Done Done Done false
3362024-09-27T03:53:56.781Z 6 Acked 1005 Write 9 Done Done Done false
3372024-09-27T03:53:56.781Z 7 Acked 1006 Read 8 Done Done Done false
3382024-09-27T03:53:56.781Z 8 Acked 1007 Write 8 Done Done Done false
3392024-09-27T03:53:56.781Z 9 Acked 1008 Read 5 Done Done Done false
3402024-09-27T03:53:56.781Z 10 Acked 1009 Read 6 Done Done Done false
3412024-09-27T03:53:56.781Z STATES DS:0 DS:1 DS:2 TOTAL
3422024-09-27T03:53:56.781Z Sent 0 0 0 0
3432024-09-27T03:53:56.781Z Done 9 9 9 27
3442024-09-27T03:53:56.787Z Skipped 0 0 0 0
3452024-09-27T03:53:56.787ZError: Difference in extent metadata found!
3462024-09-27T03:53:56.787Z Error 0 0 0 0
3472024-09-27T03:53:56.787ZLast Flush: 1000 1000 1000
3482024-09-27T03:53:56.787ZDownstairs last five completed: 1000
3492024-09-27T03:53:56.787ZUpstairs last five completed: 10 9 8 7 6
3502024-09-27T03:53:56.787Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:53:55.65741668Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1224}
3512024-09-27T03:53:56.787ZWrote out file "/var/tmp/test_up-build/verify_file"
3522024-09-27T03:53:56.787ZRepair part 1 passed
3532024-09-27T03:53:56.787Z
3542024-09-27T03:53:56.787ZKill the current downstairs
3552024-09-27T03:53:56.787Zrm -rf /var/tmp/test_up-build/8830
3562024-09-27T03:53:56.787ZNow put back the original so we have a mismatch
3572024-09-27T03:53:56.787Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
3582024-09-27T03:53:56.787ZRestart downstairs with old directory
3592024-09-27T03:53:56.787Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
3602024-09-27T03:53:56.787ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3612024-09-27T03:53:56.787Z 0 000-009 31 31 31 1699 1699 1699 T T F <---
3622024-09-27T03:53:56.787Z 1 010-019 31 31 31 1699 1699 1699 T T F <---
3632024-09-27T03:53:56.787Z 2 020-029 31 31 31 1699 1699 1699 T T F <---
3642024-09-27T03:53:56.788Z 3 030-039 31 31 31 1699 1699 1699 T T F <---
3652024-09-27T03:53:56.788Z 4 040-049 31 31 31 1699 1699 1699 F F F
3662024-09-27T03:53:56.788ZMax gen: 31, Max flush: 1699
3672024-09-27T03:53:56.788Zdump test found error as expected
3682024-09-27T03:53:56.788Z
3692024-09-27T03:53:56.788Z
3702024-09-27T03:53:56.788Z/var/tmp/bins/crutest verify --range -g 33 -q --key NOZATywXytdQgM+J/eGxj9ahhft21rSFsjcAdbvyhx4= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
3712024-09-27T03:53:56.788Z{"msg":"use region info: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.917547085Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"task":"crutest"}
3722024-09-27T03:53:56.788Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.918091228Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"task":"crutest"}
3732024-09-27T03:53:56.788Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.919751326Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"task":"crutest"}
3742024-09-27T03:53:56.788Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.919927743Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231}
3752024-09-27T03:53:56.788Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.91996149Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231}
3762024-09-27T03:53:56.788Z{"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.919978278Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231}
3772024-09-27T03:53:56.788Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.920011066Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231}
3782024-09-27T03:53:56.788Z{"msg":"Crucible 63ceff9d-2732-4956-a84a-c46cd30fbc2b has session id: 159a039d-0dac-4d0f-9438-9f647d8a2b48","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.920439078Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
3792024-09-27T03:53:56.788Z{"msg":"Upstairs opts: Upstairs UUID: 63ceff9d-2732-4956-a84a-c46cd30fbc2b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.920484897Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
3802024-09-27T03:53:56.788Z{"msg":"Crucible stats registered with UUID: 63ceff9d-2732-4956-a84a-c46cd30fbc2b","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.920512857Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
3812024-09-27T03:53:56.788Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.954444261Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231}
3822024-09-27T03:53:56.788Z{"msg":"63ceff9d-2732-4956-a84a-c46cd30fbc2b active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.954537776Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
3832024-09-27T03:53:56.788Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.954562029Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"0","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
3842024-09-27T03:53:56.788Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.95459065Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"0","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
3852024-09-27T03:53:56.788Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.954609736Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"1","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
3862024-09-27T03:53:56.788Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.954626654Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"1","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
3872024-09-27T03:53:56.788Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.954644292Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"2","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
3882024-09-27T03:53:56.788Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.954660991Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"2","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
3892024-09-27T03:53:56.788Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.954677529Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
3902024-09-27T03:53:56.788Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.954694338Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"":"io task","client":"0","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
3912024-09-27T03:53:56.788Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.954711396Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"":"io task","client":"1","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
3922024-09-27T03:53:56.788Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:53:55.954793579Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
3932024-09-27T03:53:56.788Z{"msg":"ds_connection connected from Ok(127.0.0.1:53505)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.954818032Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"":"io task","client":"0","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
3942024-09-27T03:53:56.788Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:53:55.954847081Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
3952024-09-27T03:53:56.789Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:53:55.954880988Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
3962024-09-27T03:53:56.789Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.954908909Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"2","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
3972024-09-27T03:53:56.789Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.954936819Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"2","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
3982024-09-27T03:53:56.789Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.955026857Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
3992024-09-27T03:53:56.789Z{"msg":"ds_connection connected from Ok(127.0.0.1:51629)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.955101295Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"":"io task","client":"1","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4002024-09-27T03:53:56.789Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.95548328Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"0","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4012024-09-27T03:53:56.789Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.955511191Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"0","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4022024-09-27T03:53:56.789Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.955608573Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"1","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4032024-09-27T03:53:56.789Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.95562805Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"1","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4042024-09-27T03:53:56.789Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.955957441Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"0","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4052024-09-27T03:53:56.789Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.956070163Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"1","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4062024-09-27T03:53:56.789Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.956281896Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"0","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4072024-09-27T03:53:56.789Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.956308218Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"0","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4082024-09-27T03:53:56.789Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.956322018Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4092024-09-27T03:53:56.789Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.956339376Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"1","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4102024-09-27T03:53:56.789Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.956352976Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"1","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4112024-09-27T03:53:56.789Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:55.956370165Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4122024-09-27T03:54:06.375Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.956174132Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4132024-09-27T03:54:06.380Z{"msg":"ds_connection connected from Ok(127.0.0.1:59125)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.956320691Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4142024-09-27T03:54:06.380Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.957345859Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"2","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4152024-09-27T03:54:06.380Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.957385372Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"2","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4162024-09-27T03:54:06.380Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.957959803Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"2","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4172024-09-27T03:54:06.381Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.958333924Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"2","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4182024-09-27T03:54:06.381Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.958365022Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"2","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4192024-09-27T03:54:06.381Z{"msg":"[0]R flush_numbers: [1699, 1699, 1699, 1699, 1699]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.958381131Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4202024-09-27T03:54:06.381Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.958402206Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4212024-09-27T03:54:06.381Z{"msg":"[0]R dirty: [true, true, true, true, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.958416347Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4222024-09-27T03:54:06.381Z{"msg":"[1]R flush_numbers: [1699, 1699, 1699, 1699, 1699]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.958436513Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4232024-09-27T03:54:06.381Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.958450743Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4242024-09-27T03:54:06.382Z{"msg":"[1]R dirty: [true, true, true, true, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.958464333Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4252024-09-27T03:54:06.382Z{"msg":"[2]R flush_numbers: [1699, 1699, 1699, 1699, 1699]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.958484329Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4262024-09-27T03:54:06.382Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.95849795Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4272024-09-27T03:54:06.382Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.958533695Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4282024-09-27T03:54:06.382Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.958558698Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4292024-09-27T03:54:06.382Z{"msg":"Generation requested: 33 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.958580633Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4302024-09-27T03:54:06.382Z{"msg":"Next flush: 1700","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.958593983Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4312024-09-27T03:54:06.382Z{"msg":"Extents 0 dirty","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.95861349Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4322024-09-27T03:54:06.382Z{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.95862726Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4332024-09-27T03:54:06.382Z{"msg":"extent:0 gens: 31 31 31","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.958647956Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4342024-09-27T03:54:06.382Z{"msg":"extent:0 flush: 1699 1699 1699 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.958662825Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4352024-09-27T03:54:06.382Z{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.958683761Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4362024-09-27T03:54:06.382Z{"msg":"extent:0 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.958698291Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4372024-09-27T03:54:06.382Z{"msg":"find dest for source 0 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.958718747Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4382024-09-27T03:54:06.382Z{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.958733806Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4392024-09-27T03:54:06.382Z{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.958747897Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4402024-09-27T03:54:06.382Z{"msg":"Extents 1 dirty","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.958777296Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4412024-09-27T03:54:06.382Z{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.958806566Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4422024-09-27T03:54:06.382Z{"msg":"extent:1 gens: 31 31 31","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.95884427Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4432024-09-27T03:54:06.383Z{"msg":"extent:1 flush: 1699 1699 1699 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.95887313Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4442024-09-27T03:54:06.383Z{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.958911263Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4452024-09-27T03:54:06.383Z{"msg":"extent:1 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.958950016Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4462024-09-27T03:54:06.383Z{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.958980935Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4472024-09-27T03:54:06.383Z{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.95901699Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4482024-09-27T03:54:06.383Z{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.959033378Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4492024-09-27T03:54:06.383Z{"msg":"Extents 2 dirty","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.959054924Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4502024-09-27T03:54:06.383Z{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.959068714Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4512024-09-27T03:54:06.383Z{"msg":"extent:2 gens: 31 31 31","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.959083494Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4522024-09-27T03:54:06.383Z{"msg":"extent:2 flush: 1699 1699 1699 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.959100192Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4532024-09-27T03:54:06.383Z{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.959121797Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4542024-09-27T03:54:06.383Z{"msg":"extent:2 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.959135867Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4552024-09-27T03:54:06.383Z{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.959156553Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4562024-09-27T03:54:06.383Z{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.959170783Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4572024-09-27T03:54:06.383Z{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.959191349Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4582024-09-27T03:54:06.383Z{"msg":"Extents 3 dirty","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.959205489Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4592024-09-27T03:54:06.383Z{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.959225055Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4602024-09-27T03:54:06.383Z{"msg":"extent:3 gens: 31 31 31","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.959239396Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4612024-09-27T03:54:06.383Z{"msg":"extent:3 flush: 1699 1699 1699 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.959260381Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4622024-09-27T03:54:06.383Z{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.959274821Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4632024-09-27T03:54:06.383Z{"msg":"extent:3 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.959295776Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4642024-09-27T03:54:06.383Z{"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.959310196Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4652024-09-27T03:54:06.383Z{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.959330572Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4662024-09-27T03:54:06.383Z{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.959344822Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4672024-09-27T03:54:06.383Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.959365558Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"0","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4682024-09-27T03:54:06.383Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.959379079Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"1","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4692024-09-27T03:54:06.383Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.959400014Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"2","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4702024-09-27T03:54:06.384Z{"msg":"starting reconciliation e3ac9362-8205-4300-8f40-2cf9b6ef8f7a: found 4 extents that need repair","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.959414274Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4712024-09-27T03:54:06.384Z{"msg":"Full repair list: {ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.959432232Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4722024-09-27T03:54:06.388Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 1700, gen_number: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 1700, gen_number: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 1700, gen_number: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 1700, gen_number: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.959508409Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4732024-09-27T03:54:06.388Z{"msg":"no Nexus client from DNS, aborting start notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:54:05.959532522Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"reconcile":"e3ac9362-8205-4300-8f40-2cf9b6ef8f7a","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4742024-09-27T03:54:06.388Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:54:05.959547951Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"reconcile":"e3ac9362-8205-4300-8f40-2cf9b6ef8f7a","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4752024-09-27T03:54:06.388Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:54:05.960345968Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"reconcile":"e3ac9362-8205-4300-8f40-2cf9b6ef8f7a","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4762024-09-27T03:54:06.388Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.960651715Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"1","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4772024-09-27T03:54:06.388Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.960693746Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"2","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4782024-09-27T03:54:06.388Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:54:05.960736307Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"reconcile":"e3ac9362-8205-4300-8f40-2cf9b6ef8f7a","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4792024-09-27T03:54:06.397Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:54:05.976699049Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"reconcile":"e3ac9362-8205-4300-8f40-2cf9b6ef8f7a","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4802024-09-27T03:54:06.400Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:54:05.97707293Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"reconcile":"e3ac9362-8205-4300-8f40-2cf9b6ef8f7a","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4812024-09-27T03:54:06.400Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:54:05.97831222Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"reconcile":"e3ac9362-8205-4300-8f40-2cf9b6ef8f7a","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4822024-09-27T03:54:06.400Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.978647437Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"1","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4832024-09-27T03:54:06.401Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.978685561Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"2","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4842024-09-27T03:54:06.402Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:54:05.978703729Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"reconcile":"e3ac9362-8205-4300-8f40-2cf9b6ef8f7a","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4852024-09-27T03:54:06.406Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:54:05.987748142Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"reconcile":"e3ac9362-8205-4300-8f40-2cf9b6ef8f7a","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4862024-09-27T03:54:06.409Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:54:05.988161126Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"reconcile":"e3ac9362-8205-4300-8f40-2cf9b6ef8f7a","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4872024-09-27T03:54:06.409Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:54:05.989198476Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"reconcile":"e3ac9362-8205-4300-8f40-2cf9b6ef8f7a","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4882024-09-27T03:54:06.410Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.989512208Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"1","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4892024-09-27T03:54:06.410Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:05.989555538Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"2","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4902024-09-27T03:54:06.410Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:54:05.989583479Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"reconcile":"e3ac9362-8205-4300-8f40-2cf9b6ef8f7a","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4912024-09-27T03:54:06.419Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:54:06.000229561Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"reconcile":"e3ac9362-8205-4300-8f40-2cf9b6ef8f7a","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4922024-09-27T03:54:06.422Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:54:06.000672923Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"reconcile":"e3ac9362-8205-4300-8f40-2cf9b6ef8f7a","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4932024-09-27T03:54:06.422Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:54:06.001693995Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"reconcile":"e3ac9362-8205-4300-8f40-2cf9b6ef8f7a","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4942024-09-27T03:54:06.422Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:06.001969344Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"1","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4952024-09-27T03:54:06.422Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:06.002020818Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"2","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4962024-09-27T03:54:06.422Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:54:06.002076799Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"reconcile":"e3ac9362-8205-4300-8f40-2cf9b6ef8f7a","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4972024-09-27T03:54:06.430Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:54:06.010872036Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"reconcile":"e3ac9362-8205-4300-8f40-2cf9b6ef8f7a","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4982024-09-27T03:54:06.433Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:06.011170639Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
4992024-09-27T03:54:06.434Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:06.011215957Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"0","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
5002024-09-27T03:54:06.434Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:06.011235943Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"1","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
5012024-09-27T03:54:06.434Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:06.011279033Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"client":"2","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
5022024-09-27T03:54:06.434Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:06.01129763Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
5032024-09-27T03:54:06.434Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:06.011323322Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
5042024-09-27T03:54:06.434Z{"msg":"63ceff9d-2732-4956-a84a-c46cd30fbc2b is now active with session: 159a039d-0dac-4d0f-9438-9f647d8a2b48","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:06.011340411Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
5052024-09-27T03:54:06.434ZWait for a query_work_queue command to finish before sending IO
5062024-09-27T03:54:06.434Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:06.011362845Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231}
5072024-09-27T03:54:06.434Z{"msg":"no Nexus client from DNS, aborting finish notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:54:06.011391515Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231,"reconcile":"e3ac9362-8205-4300-8f40-2cf9b6ef8f7a","":"downstairs","":"downstairs","session_id":"159a039d-0dac-4d0f-9438-9f647d8a2b48"}
5082024-09-27T03:54:06.434Zregion info:
5092024-09-27T03:54:06.434Z block size: 4096 bytes
5102024-09-27T03:54:06.434Z sub_volume 0 blocks / extent: 10
5112024-09-27T03:54:06.434Z sub_volume 0 extent size: 40 KiB
5122024-09-27T03:54:06.434Z sub_volume 0 extent count: 5
5132024-09-27T03:54:06.434Z total blocks: 50
5142024-09-27T03:54:06.435Z total size: 200 KiB
5152024-09-27T03:54:06.435Z encryption: yes
5162024-09-27T03:54:06.435ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
5172024-09-27T03:54:06.435ZRead and Verify all blocks (0..50 range:true)
5182024-09-27T03:54:06.472ZWrote out file "/var/tmp/test_up-build/verify_file"
5192024-09-27T03:54:06.473ZVerify test completed
5202024-09-27T03:54:06.478ZWrote out file "/var/tmp/test_up-build/verify_file"
5212024-09-27T03:54:06.478ZCLIENT: Tests done. All submitted work has been ACK'd
5222024-09-27T03:54:06.478Z----------------------------------------------------------------
5232024-09-27T03:54:06.478Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
5242024-09-27T03:54:06.478ZStates: Active Active Active
5252024-09-27T03:54:06.478ZGW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
5262024-09-27T03:54:06.478Z 1 Acked 1000 Read 50 Done Done Done false
5272024-09-27T03:54:06.478Z STATES DS:0 DS:1 DS:2 TOTAL
5282024-09-27T03:54:06.479Z Sent 0 0 0 0
5292024-09-27T03:54:06.479Z Done 1 1 1 3
5302024-09-27T03:54:06.479Z Skipped 0 0 0 0
5312024-09-27T03:54:06.479Z Error 0 0 0 0
5322024-09-27T03:54:06.479ZLast Flush: 0 0 0
5332024-09-27T03:54:06.479ZDownstairs last five completed:
5342024-09-27T03:54:06.479ZUpstairs last five completed: 1
5352024-09-27T03:54:06.479ZCLIENT: Up:0 ds:1 act:3
5362024-09-27T03:54:10.477Z----------------------------------------------------------------
5372024-09-27T03:54:10.477Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
5382024-09-27T03:54:10.477ZDownstairs last five completed: 1001 1000
5392024-09-27T03:54:10.477ZUpstairs last five completed: 2 1
5402024-09-27T03:54:10.477ZCLIENT: Up:0 ds:0 act:3
5412024-09-27T03:54:10.477ZCLIENT: All crucible jobs finished, exiting program
5422024-09-27T03:54:10.484Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:54:10.055090843Z","hostname":"w-01J8RSHJ0DQD1FAYE3AWJ6AGZ9","pid":1231}
5432024-09-27T03:54:10.492ZRepair part 2 passed
5442024-09-27T03:54:10.492Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
5452024-09-27T03:54:10.503ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
5462024-09-27T03:54:10.503Z 0 000-009 32 32 32 1700 1700 1700 F F F
5472024-09-27T03:54:10.513Z 1 010-019 32 32 32 1700 1700 1700 F F F
5482024-09-27T03:54:10.513Z 2 020-029 32 32 32 1700 1700 1700 F F F
5492024-09-27T03:54:10.513Z 3 030-039 32 32 32 1700 1700 1700 F F F
5502024-09-27T03:54:10.513Z 4 040-049 31 31 31 1699 1699 1699 F F F
5512024-09-27T03:54:10.513ZMax gen: 32, Max flush: 1700
5522024-09-27T03:54:10.513Zdump test passed
5532024-09-27T03:54:10.513Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
5542024-09-27T03:54:10.525Z Extent 1
5552024-09-27T03:54:10.529ZGEN 32 32 32
5562024-09-27T03:54:10.529ZFLUSH_ID 1700 1700 1700
5572024-09-27T03:54:10.529ZDIRTY
5582024-09-27T03:54:10.529Z
5592024-09-27T03:54:10.529ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
5602024-09-27T03:54:10.529Z010 A A A A A A
5612024-09-27T03:54:10.529Z011 A A A A A A
5622024-09-27T03:54:10.529Z012 A A A A A A
5632024-09-27T03:54:10.529Z013 A A A A A A
5642024-09-27T03:54:10.529Z014 A A A A A A
5652024-09-27T03:54:10.532Z015 A A A A A A
5662024-09-27T03:54:10.532Z016 A A A A A A
5672024-09-27T03:54:10.532Z017 A A A A A A
5682024-09-27T03:54:10.532Z018 A A A A A A
5692024-09-27T03:54:10.537Z019 A A A A A A
5702024-09-27T03:54:10.540Zdump extent test passed
5712024-09-27T03:54:10.540Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
5722024-09-27T03:54:10.553ZExtent 2 Block in extent 0 Actual block 20
5732024-09-27T03:54:10.553Z
5742024-09-27T03:54:10.562Z DATA SHA256 VER
5752024-09-27T03:54:10.563Z------ ---------------------------------------------------------------- ---
5762024-09-27T03:54:10.563Z 0 654fd14d7ed5bb152badf18a743a3af4ec4e94bb97c51e45517fd584cc1f3d42 A
5772024-09-27T03:54:10.563Z 1 654fd14d7ed5bb152badf18a743a3af4ec4e94bb97c51e45517fd584cc1f3d42 A
5782024-09-27T03:54:10.563Z 2 654fd14d7ed5bb152badf18a743a3af4ec4e94bb97c51e45517fd584cc1f3d42 A
5792024-09-27T03:54:10.563Z
5802024-09-27T03:54:10.563ZNONCES 0 1 2 DIFF
5812024-09-27T03:54:10.563Z------ ------------------------ ------------------------ ------------------------ -----
5822024-09-27T03:54:10.563Z 0 d24c8711457a9a82f6a79e66 d24c8711457a9a82f6a79e66 d24c8711457a9a82f6a79e66
5832024-09-27T03:54:10.563Z
5842024-09-27T03:54:10.563Z TAGS 0 1 2 DIFF
5852024-09-27T03:54:10.563Z------ -------------------------------- -------------------------------- -------------------------------- -----
5862024-09-27T03:54:10.563Z 0 e1fd9dd4a37506381e43d5969d730335 e1fd9dd4a37506381e43d5969d730335 e1fd9dd4a37506381e43d5969d730335
5872024-09-27T03:54:10.564Z
5882024-09-27T03:54:10.564ZHASHES 0 1 2 DIFF
5892024-09-27T03:54:10.565Z------ ---------------- ---------------- ---------------- -----
5902024-09-27T03:54:10.565Z
5912024-09-27T03:54:10.577Zdump block test passed
5922024-09-27T03:54:10.577ZUpstairs tests have completed, stopping all downstairs
5932024-09-27T03:54:15.624ZCreating four larger downstairs regions
5942024-09-27T03:54:16.092ZStarting four downstairs
5952024-09-27T03:54:21.096ZNow do the replace-reconcile test
5962024-09-27T03:56:56.114ZCompleted test: replace-reconcile
5972024-09-27T03:56:56.116ZNow do the replace-before-active test
5982024-09-27T03:59:40.562ZCompleted test: replace-before-active
5992024-09-27T03:59:40.565ZTests have completed, stopping all downstairs
6002024-09-27T03:59:40.621Z
6012024-09-27T03:59:40.621ZAll Tests have passed
6022024-09-27T03:59:40.624Z9:17 Test duration
6032024-09-27T03:59:40.624Z
6042024-09-27T03:59:40.624Zreal 9:17.053495086
6052024-09-27T03:59:40.624Zuser 3:37.905344017
6062024-09-27T03:59:40.624Zsys 13.242370407
6072024-09-27T03:59:40.624Ztrap 0.204126945
6082024-09-27T03:59:40.624Ztflt 0.014018123
6092024-09-27T03:59:40.624Zdflt 0.070308606
6102024-09-27T03:59:40.624Zkflt 0.000996555
6112024-09-27T03:59:40.624Zlock 8:28:44.384425442
6122024-09-27T03:59:40.624Zslp 33:08.768859657
6132024-09-27T03:59:40.624Zlat 11.614485035
6142024-09-27T03:59:40.624Zstop 0.126243456
6152024-09-27T03:59:40.624Z+ echo 'test-up-encrypted ends'
6162024-09-27T03:59:40.624Ztest-up-encrypted ends
6172024-09-27T03:59:45.631Zprocess exited: duration 565320 ms, exit code 0
6182024-09-27T03:59:45.631Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
6192024-09-27T04:00:45.712Zexec warning: : stdout descriptor may be held open by a background process; giving up!
6202024-09-27T04:00:45.713Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
6212024-09-27T04:00:45.724Zfound 8 output files
6222024-09-27T04:00:45.724Zuploading: /tmp/debug/df.txt (1270 bytes)
6232024-09-27T04:00:46.736Zuploaded: /tmp/debug/df.txt
6242024-09-27T04:00:46.736Zuploading: /tmp/debug/dtrace.txt (491306 bytes)
6252024-09-27T04:00:47.758Zuploaded: /tmp/debug/dtrace.txt
6262024-09-27T04:00:47.758Zuploading: /tmp/debug/iostat.txt (188205 bytes)
6272024-09-27T04:00:47.767Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 188205 -> 188809
6282024-09-27T04:00:48.774Zuploaded: /tmp/debug/iostat.txt
6292024-09-27T04:00:48.774Zuploading: /tmp/debug/mpstat.txt (455418 bytes)
6302024-09-27T04:00:48.786Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 455418 -> 457611
6312024-09-27T04:00:49.795Zuploaded: /tmp/debug/mpstat.txt
6322024-09-27T04:00:49.795Zuploading: /tmp/debug/paging.txt (82845 bytes)
6332024-09-27T04:00:49.803Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 82845 -> 83313
6342024-09-27T04:00:50.809Zuploaded: /tmp/debug/paging.txt
6352024-09-27T04:00:50.809Zuploading: /tmp/debug/prstat.txt (876155 bytes)
6362024-09-27T04:00:50.825Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 876155 -> 883910
6372024-09-27T04:00:51.832Zuploaded: /tmp/debug/prstat.txt
6382024-09-27T04:00:51.832Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
6392024-09-27T04:00:52.840Zuploaded: /tmp/debug/psrinfo.txt
6402024-09-27T04:00:52.840Zuploading: /tmp/debug/upstairs-info.txt (93062 bytes)
6412024-09-27T04:00:53.854Zuploaded: /tmp/debug/upstairs-info.txt